overvues 0.0.1 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +58 -2
  2. package/dist/ansi.js +24 -0
  3. package/dist/attach-client.js +586 -0
  4. package/dist/auth-connect-browser.js +112 -0
  5. package/dist/auth-connect.js +306 -0
  6. package/dist/browser-bootstrap.js +190 -0
  7. package/dist/browser-loopback.js +289 -0
  8. package/dist/build-identity.js +6 -0
  9. package/dist/codex-reviewer-apparmor.json +4 -0
  10. package/dist/codex-reviewer-seccomp.json +1238 -0
  11. package/dist/codex-runtime-protocol.js +1 -0
  12. package/dist/codex-subscription.js +53 -0
  13. package/dist/config.js +297 -0
  14. package/dist/connect-comparison-code.js +26 -0
  15. package/dist/development-runner.js +109 -0
  16. package/dist/dns-doctor.js +390 -0
  17. package/dist/docker-names.js +47 -0
  18. package/dist/doctor.js +114 -0
  19. package/dist/gadget-job.js +804 -0
  20. package/dist/health-check.js +112 -0
  21. package/dist/help.js +71 -0
  22. package/dist/host-auth.js +22 -0
  23. package/dist/host-diagnostics.js +20 -0
  24. package/dist/host-runtime-identity.js +39 -0
  25. package/dist/host-runtime-metadata.js +56 -0
  26. package/dist/host-service.js +377 -0
  27. package/dist/host-stack.js +484 -0
  28. package/dist/host-supervisor.js +237 -0
  29. package/dist/host-update.js +387 -0
  30. package/dist/hub-address.js +100 -0
  31. package/dist/hub-client.js +459 -0
  32. package/dist/hub-kind.js +88 -0
  33. package/dist/identity.js +73 -0
  34. package/dist/index.js +185 -0
  35. package/dist/init.js +295 -0
  36. package/dist/isolated-run-cache.js +116 -0
  37. package/dist/local-stack.js +685 -0
  38. package/dist/machine-capacity.js +128 -0
  39. package/dist/managed-runner-image.js +13 -0
  40. package/dist/os-acted-record.js +49 -0
  41. package/dist/os-adapter.js +8190 -0
  42. package/dist/os-command.js +292 -0
  43. package/dist/os-handoff.js +274 -0
  44. package/dist/os-keychain.js +247 -0
  45. package/dist/os-machine.js +818 -0
  46. package/dist/os-owner.js +61 -0
  47. package/dist/os-private-file.js +98 -0
  48. package/dist/os-protocol.js +468 -0
  49. package/dist/os-session-store.js +133 -0
  50. package/dist/os-status.js +102 -0
  51. package/dist/picker.js +863 -0
  52. package/dist/portable-workspace-capability.js +60 -0
  53. package/dist/project-environment-capability.js +62 -0
  54. package/dist/project-environment.js +28 -0
  55. package/dist/project-preparation.js +134 -0
  56. package/dist/readiness.js +28 -0
  57. package/dist/redaction.js +27 -0
  58. package/dist/reviewer-isolation-capability.js +61 -0
  59. package/dist/runner-api-auth.js +80 -0
  60. package/dist/runner-command-fence.js +536 -0
  61. package/dist/runner-inspection.js +57 -0
  62. package/dist/runner-inventory.js +587 -0
  63. package/dist/runner-server.js +2520 -0
  64. package/dist/runner-session.js +470 -0
  65. package/dist/runner-slug-folder.js +39 -0
  66. package/dist/runner-slug.js +40 -0
  67. package/dist/setup.js +139 -0
  68. package/dist/shutdown.js +102 -0
  69. package/dist/splash.js +65 -0
  70. package/dist/staged-workspaces.js +441 -0
  71. package/dist/staging-disk.js +97 -0
  72. package/dist/terminal-lane.js +339 -0
  73. package/dist/terminal-recovery.js +502 -0
  74. package/licenses/capnweb.txt +23 -0
  75. package/licenses/moby-profiles.txt +212 -0
  76. package/licenses/ws.txt +22 -0
  77. package/package.json +28 -7
  78. package/cli.js +0 -2
@@ -0,0 +1,2520 @@
1
+ import http from "node:http";
2
+ import { randomBytes, randomUUID } from "node:crypto";
3
+ import { HOST_PACKAGE_VERSION, readRuntimeImagePlatform } from "./host-runtime-identity.js";
4
+ // Moby's default syscall policy plus the namespace operations used by Bubblewrap.
5
+ // Provenance and Apache-2.0 license: ../licenses/moby-profiles.txt.
6
+ import codexReviewerSeccomp from "./codex-reviewer-seccomp.json" with { type: "json" };
7
+ import codexReviewerAppArmor from "./codex-reviewer-apparmor.json" with { type: "json" };
8
+ import { spawn } from "node:child_process";
9
+ import { createHash } from "node:crypto";
10
+ import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
11
+ import { tmpdir } from "node:os";
12
+ import { basename, dirname, join, resolve } from "node:path";
13
+ import { HUB_URL, LOCAL_STATE_DIR, hubControlHeaders, parseByteSize } from "./config.js";
14
+ import { DevelopmentHubGate, HARNESS_DIST_CONTAINER_PATH, HARNESS_OVERLAY_REFUSED, STAGING_HOLD_DEFAULT_MS, StagingHold, fetchHubBuildChannel, readDevelopmentRunnerOptions, validateHarnessOverlayDir, } from "./development-runner.js";
15
+ import { redactEnvValues } from "./redaction.js";
16
+ import { inspectRunnerState } from "./runner-inspection.js";
17
+ import { RunnerCommandFenceError, RunnerCommandFenceStore, RUNNER_COMMAND_PROTOCOL, } from "./runner-command-fence.js";
18
+ import { closeHttpServer } from "./shutdown.js";
19
+ import { CODEX_RUNTIME_AUTH_PROTOCOL } from "./codex-runtime-protocol.js";
20
+ import { validateProjectEnvironmentDelivery } from "./project-environment.js";
21
+ import { requireProjectEnvironmentContainer, requireProjectEnvironmentImage, resolveProjectEnvironmentImage } from "./project-environment-capability.js";
22
+ import { GADGET_MAP_BACK_IMAGE_LABEL, GADGET_TOOLS_IMAGE_LABEL, GadgetJobRequestError, GadgetJobs, parseGadgetJobRequest, parseGadgetMapBackRequest, readGadgetJobBody } from "./gadget-job.js";
23
+ import { PORTABLE_NATIVE_LABEL, requirePortableWorkspaceContainer, requirePortableWorkspaceImage, resolvePortableWorkspaceImage } from "./portable-workspace-capability.js";
24
+ import { parseManagedLocalRunnerImageSourceId } from "./managed-runner-image.js";
25
+ import { resolveHostReviewerIsolationProtocol } from "./reviewer-isolation-capability.js";
26
+ import { StagedWorkspaces, STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES, } from "./staged-workspaces.js";
27
+ import { createDockerDiskProbe, decideStagingAtStartup, stageRunDiskRefusal, } from "./staging-disk.js";
28
+ import { RunnerInventory, emptyDockerVolume, readRunnerInventory, reconcileRunnerVolumes, } from "./runner-inventory.js";
29
+ import { DOCKER_INFO_TTL_MS, TtlSingleFlight, VolumePresenceCache } from "./isolated-run-cache.js";
30
+ import { MIRROR_REFRESH_CONTAINER_TIMEOUT_MS, PREPARE_PROJECT_SECRET_NAMES, ProjectPreparationRequestError, buildMirrorRefreshEnv, buildMirrorRefreshRunArgs, parseMirrorRefreshOutcome, parseProjectPreparationRequest, } from "./project-preparation.js";
31
+ import { dockerLabel, dockerName } from "./docker-names.js";
32
+ import { ISOLATED_RUN_LABEL_KEY, countRunningEnvironments, isMachineFull, runCapacityDocker } from "./machine-capacity.js";
33
+ import { createRunnerApiToken, isAuthorizedRunnerRequest, removeRunnerApiTokenFile, writeRunnerApiTokenFile, } from "./runner-api-auth.js";
34
+ import { assertValidRunnerSlug, isValidRunnerSlug, RunnerSlugError } from "./runner-slug.js";
35
+ import { caseSafeSlugFolderName, findExactDirectoryEntry } from "./runner-slug-folder.js";
36
+ import { handleTerminalLaneUpgrade, hasRunningTerminalLane, openDockerExecRelay, parseTerminalLaneTicketRequest, TERMINAL_LANE_CONTAINER_HOST, TERMINAL_LANE_CONTAINER_PORT, TERMINAL_LANE_INSPECT_FORMAT, TerminalLaneRelays, TerminalLaneTickets, } from "./terminal-lane.js";
37
+ import { CLI, envVar, OS } from "./identity.js";
38
+ function containerName(slug) {
39
+ return dockerName(slug.replace(/[^a-zA-Z0-9_.-]/g, "-"));
40
+ }
41
+ /**
42
+ * Binding loopback keeps other machines out, and the API secret
43
+ * (`runner-api-auth.ts`) keeps out this machine's containers, which reach host
44
+ * loopback through `host.docker.internal` on Docker Desktop. A web page in a
45
+ * browser on this machine can still send a CORS "simple" request (no
46
+ * preflight, e.g. `Content-Type: text/plain`); it cannot know the secret, but
47
+ * as a second line every mutating request must also come without an Origin
48
+ * and as JSON, which only a non-browser client can send unprompted.
49
+ */
50
+ function isRefusedBrowserMutation(req) {
51
+ if (req.method === "GET" || req.method === "HEAD")
52
+ return false;
53
+ if (req.headers.origin !== undefined)
54
+ return true;
55
+ const mediaType = String(req.headers["content-type"] ?? "").split(";")[0].trim().toLowerCase();
56
+ return mediaType !== "application/json";
57
+ }
58
+ function json(res, status, body) {
59
+ res.writeHead(status, { "Content-Type": "application/json" });
60
+ res.end(JSON.stringify(body));
61
+ }
62
+ function notFound(res) {
63
+ json(res, 404, { error: "Not found" });
64
+ }
65
+ function methodNotAllowed(res) {
66
+ json(res, 405, { error: "Method not allowed" });
67
+ }
68
+ /** A Create or Start refused before it was accepted: this machine runs as many environments as its limit allows. */
69
+ export class MachineFullError extends Error {
70
+ runningEnvironments;
71
+ environmentLimit;
72
+ status = 409;
73
+ code = "machine_full";
74
+ constructor(runningEnvironments, environmentLimit) {
75
+ super(`This machine is full (${runningEnvironments} of ${environmentLimit} environments running). Stop an environment or raise the limit (${envVar("MAX_ENVIRONMENTS")}).`);
76
+ this.runningEnvironments = runningEnvironments;
77
+ this.environmentLimit = environmentLimit;
78
+ this.name = "MachineFullError";
79
+ }
80
+ }
81
+ function requestError(res, error, fallbackStatus = 400) {
82
+ if (error instanceof MachineFullError) {
83
+ json(res, error.status, {
84
+ error: error.message,
85
+ code: error.code,
86
+ runningEnvironments: error.runningEnvironments,
87
+ environmentLimit: error.environmentLimit,
88
+ });
89
+ return;
90
+ }
91
+ if (error instanceof RunnerSlugError) {
92
+ json(res, error.status, { error: error.message, code: error.code });
93
+ return;
94
+ }
95
+ if (error instanceof RunnerCommandFenceError) {
96
+ json(res, error.status, {
97
+ error: error.message,
98
+ code: error.code,
99
+ ...(error.currentCommandGeneration !== undefined
100
+ ? { currentCommandGeneration: error.currentCommandGeneration }
101
+ : {}),
102
+ });
103
+ return;
104
+ }
105
+ json(res, fallbackStatus, { error: error instanceof Error ? error.message : String(error) });
106
+ }
107
+ function readJson(req) {
108
+ return new Promise((resolve, reject) => {
109
+ let body = "";
110
+ req.on("data", (chunk) => {
111
+ body += chunk;
112
+ if (body.length > 1024 * 1024) {
113
+ reject(new Error("Request body too large"));
114
+ req.destroy();
115
+ }
116
+ });
117
+ req.on("end", () => {
118
+ if (!body) {
119
+ resolve({});
120
+ return;
121
+ }
122
+ try {
123
+ resolve(JSON.parse(body));
124
+ }
125
+ catch {
126
+ reject(new Error("Invalid JSON"));
127
+ }
128
+ });
129
+ req.on("error", reject);
130
+ });
131
+ }
132
+ function runDocker(args) {
133
+ return new Promise((resolve, reject) => {
134
+ const child = spawn("docker", args, {
135
+ stdio: ["ignore", "pipe", "pipe"],
136
+ });
137
+ let stdout = "";
138
+ let stderr = "";
139
+ child.stdout.on("data", (chunk) => {
140
+ stdout += chunk.toString();
141
+ });
142
+ child.stderr.on("data", (chunk) => {
143
+ stderr += chunk.toString();
144
+ });
145
+ child.on("error", reject);
146
+ child.on("close", (code) => {
147
+ if (code === 0) {
148
+ resolve(stdout.trim());
149
+ return;
150
+ }
151
+ const error = new Error(stderr.trim() || stdout.trim() || `docker ${args.join(" ")} failed`);
152
+ error.code = String(code);
153
+ reject(error);
154
+ });
155
+ });
156
+ }
157
+ /**
158
+ * Whether Docker has ever started this container. Derived from Docker rather
159
+ * than from a flag the runner writes, so it survives a runner crash between
160
+ * recording a launch and starting the container.
161
+ */
162
+ function hasRun(info) {
163
+ if (!info)
164
+ return false;
165
+ if (info.State?.Status === "created")
166
+ return false;
167
+ const startedAt = info.State?.StartedAt;
168
+ if (!startedAt)
169
+ return false;
170
+ const parsed = Date.parse(startedAt);
171
+ return Number.isFinite(parsed) && parsed > 0;
172
+ }
173
+ /**
174
+ * A container Docker created but never started: its state is still `created`
175
+ * (Docker never returns a container to that state once it has run) and it has
176
+ * no start time. No process in it ever ran.
177
+ */
178
+ function neverStarted(info) {
179
+ if (info?.State?.Status !== "created")
180
+ return false;
181
+ const startedAt = Date.parse(info.State.StartedAt ?? "");
182
+ return !(Number.isFinite(startedAt) && startedAt > 0);
183
+ }
184
+ function startedBefore(info, isoTimestamp) {
185
+ const startedAt = info?.State?.StartedAt;
186
+ if (!startedAt)
187
+ return false;
188
+ const started = Date.parse(startedAt);
189
+ const reference = Date.parse(isoTimestamp);
190
+ return Number.isFinite(started) && Number.isFinite(reference) && started <= reference;
191
+ }
192
+ async function inspectContainer(slug, strict = false) {
193
+ try {
194
+ // Typed: for a missing container an untyped `docker inspect` goes on to
195
+ // look the name up as an image, network and volume, and the volume lookup
196
+ // waits for any volume removal in progress (Docker's local volume driver
197
+ // holds one lock while it deletes a volume's files). Removing a workspace
198
+ // with dependencies held a fresh slug's create for 10 to 20 s.
199
+ const output = await runDocker(["inspect", "--type", "container", containerName(slug)]);
200
+ const parsed = JSON.parse(output);
201
+ if (strict && (!Array.isArray(parsed) || parsed.length !== 1 || typeof parsed[0]?.State?.Status !== "string" || !parsed[0].State.Status.trim())) {
202
+ throw new Error("Docker returned an invalid container inspection.");
203
+ }
204
+ return parsed[0] ?? null;
205
+ }
206
+ catch (err) {
207
+ const message = err instanceof Error ? err.message : String(err);
208
+ if (/no such (?:object|container)/i.test(message)) {
209
+ return null;
210
+ }
211
+ throw err;
212
+ }
213
+ }
214
+ function getStatus(info) {
215
+ const status = info?.State?.Status;
216
+ if (!status)
217
+ return "stopped";
218
+ if (status === "exited")
219
+ return "stopped";
220
+ return status;
221
+ }
222
+ async function forceRemoveContainer(slug) {
223
+ try {
224
+ await runDocker(["rm", "-f", containerName(slug)]);
225
+ return true;
226
+ }
227
+ catch (err) {
228
+ const message = err instanceof Error ? err.message : String(err);
229
+ if (message.includes("No such container")) {
230
+ return false;
231
+ }
232
+ throw err;
233
+ }
234
+ }
235
+ async function removeVolumeIfPresent(volume, image) {
236
+ // Emptied from a container first, so the removal does not hold Docker's
237
+ // volume lock (and with it every other create) for the length of the delete.
238
+ await emptyDockerVolume(runDocker, image, volume);
239
+ try {
240
+ await runDocker(["volume", "rm", volume]);
241
+ return true;
242
+ }
243
+ catch (err) {
244
+ const message = (err instanceof Error ? err.message : String(err)).toLowerCase();
245
+ if (message.includes("no such volume"))
246
+ return false;
247
+ throw err;
248
+ }
249
+ }
250
+ /**
251
+ * Removes the slug's restore cache, the workspace volume it recorded (an
252
+ * adopted staged volume keeps its own name) and the default-named one, which
253
+ * an older runner may have used before the record existed.
254
+ */
255
+ async function removeWorkspaceVolumes(slug, recorded, image) {
256
+ await removeVolumeIfPresent(`${RESTORE_CACHE_VOLUME_PREFIX}${slug}`, image);
257
+ const fallback = workspaceVolumeName(slug);
258
+ let removed = false;
259
+ if (recorded && recorded !== fallback)
260
+ removed = await removeVolumeIfPresent(recorded, image);
261
+ return (await removeVolumeIfPresent(fallback, image)) || removed;
262
+ }
263
+ async function removeStoppedContainer(slug) {
264
+ try {
265
+ await runDocker(["rm", containerName(slug)]);
266
+ return true;
267
+ }
268
+ catch (err) {
269
+ const message = err instanceof Error ? err.message : String(err);
270
+ if (message.includes("No such container"))
271
+ return false;
272
+ throw err;
273
+ }
274
+ }
275
+ export const HOST_OPENCODE_STATE_MOUNT_PATH = "/home/tiller/.local/share/opencode";
276
+ const START_OP_ID_ENV_VAR = "TILLER_LIFECYCLE_START_OP_ID";
277
+ const HOST_COMMAND_GENERATION_ENV_VAR = "TILLER_HOST_COMMAND_GENERATION";
278
+ const HOST_COMMAND_FENCE_REQUIRED_ENV_VAR = "TILLER_HOST_COMMAND_FENCE_REQUIRED";
279
+ const HOST_COMMAND_FENCE_CONTAINER_PATH = "/run/tiller-host-command";
280
+ const HOST_COMMAND_GENERATION_LABEL = dockerLabel("command-generation");
281
+ const HOST_OPERATION_ID_LABEL = dockerLabel("operation-id");
282
+ const HOST_CONFIG_HASH_LABEL = dockerLabel("config-hash");
283
+ const RESTORE_CACHE_VOLUME_PREFIX = dockerName("restore-");
284
+ const WORKSPACE_VOLUME_PREFIX = dockerName("ws-");
285
+ const WORKSPACE_CONTAINER_PATH = "/workspace";
286
+ const PACKAGE_CACHE_ROOT = "/var/cache/tiller";
287
+ // Shared across every environment container on this machine. These hold public
288
+ // package archives, never workspace or credential state, and they live outside
289
+ // /home/tiller/.cache because the OpenCode seed step chowns that tree on every
290
+ // boot and would traverse them.
291
+ const PACKAGE_CACHE_MOUNTS = [
292
+ {
293
+ volume: dockerName("cache-npm"),
294
+ target: `${PACKAGE_CACHE_ROOT}/npm`,
295
+ env: { npm_config_cache: `${PACKAGE_CACHE_ROOT}/npm` },
296
+ },
297
+ {
298
+ volume: dockerName("cache-pnpm"),
299
+ target: `${PACKAGE_CACHE_ROOT}/pnpm`,
300
+ env: { npm_config_store_dir: `${PACKAGE_CACHE_ROOT}/pnpm` },
301
+ },
302
+ {
303
+ volume: dockerName("cache-yarn"),
304
+ target: `${PACKAGE_CACHE_ROOT}/yarn`,
305
+ env: { YARN_CACHE_FOLDER: `${PACKAGE_CACHE_ROOT}/yarn` },
306
+ },
307
+ {
308
+ volume: dockerName("cache-cargo"),
309
+ target: `${PACKAGE_CACHE_ROOT}/cargo`,
310
+ env: { CARGO_HOME: `${PACKAGE_CACHE_ROOT}/cargo` },
311
+ },
312
+ {
313
+ volume: dockerName("cache-go"),
314
+ target: `${PACKAGE_CACHE_ROOT}/go`,
315
+ env: { GOMODCACHE: `${PACKAGE_CACHE_ROOT}/go` },
316
+ },
317
+ {
318
+ volume: dockerName("cache-pip"),
319
+ target: `${PACKAGE_CACHE_ROOT}/pip`,
320
+ env: { PIP_CACHE_DIR: `${PACKAGE_CACHE_ROOT}/pip` },
321
+ },
322
+ ];
323
+ export function workspaceVolumeName(slug) {
324
+ return `${WORKSPACE_VOLUME_PREFIX}${slug.replace(/[^a-zA-Z0-9_.-]/g, "-")}`;
325
+ }
326
+ // The image creates this directory owned by tiller-mirror (uid 1002) after the
327
+ // cache chown, so a new mirror volume is writable by that account only: the
328
+ // privileged bootstrap fills it as tiller-mirror and every workload reads it.
329
+ export const GIT_MIRROR_CONTAINER_PATH = `${PACKAGE_CACHE_ROOT}/git-mirror`;
330
+ const GIT_MIRROR_VOLUME_PREFIX = dockerName("mirror-");
331
+ const GIT_MIRROR_ENV_VAR = "TILLER_GIT_MIRROR_PATH";
332
+ export { GIT_MIRROR_IDLE_LIMIT_MS } from "./runner-inventory.js";
333
+ /** The canonical lowercase `https://github.com/<owner>/<repo>.git`, or null. */
334
+ export function canonicalGitHubRepositoryUrl(repoUrl) {
335
+ // Same address rule as the harness's frozenCheckoutRepository.
336
+ const match = /^https:\/\/github\.com\/([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?\/([a-z0-9._-]+?))(?:\.git)?$/i.exec(repoUrl);
337
+ if (!match || [".", ".."].includes(match[2]))
338
+ return null;
339
+ return `https://github.com/${match[1].toLowerCase()}.git`;
340
+ }
341
+ /** One mirror volume per repository on this machine; null for non-GitHub repositories. */
342
+ export function mirrorVolumeName(repoUrl) {
343
+ const canonical = canonicalGitHubRepositoryUrl(repoUrl);
344
+ if (!canonical)
345
+ return null;
346
+ return `${GIT_MIRROR_VOLUME_PREFIX}${createHash("sha256").update(canonical).digest("hex").slice(0, 16)}`;
347
+ }
348
+ /** The mirrors in the runner inventory, by volume. */
349
+ export function readGitMirrorRecords(localStateDir = LOCAL_STATE_DIR) {
350
+ return Object.fromEntries(Object.entries(readRunnerInventory(localStateDir)).flatMap(([volume, entry]) => entry.kind === "mirror" && entry.repoUrl ? [[volume, { repoUrl: entry.repoUrl, lastUsedAt: entry.lastUsedAt }]] : []));
351
+ }
352
+ export function recordGitMirrorUse(volume, repoUrl, localStateDir = LOCAL_STATE_DIR, now = Date.now()) {
353
+ new RunnerInventory(localStateDir, () => now).recordMirrorUse(volume, repoUrl);
354
+ }
355
+ async function volumeExists(volume) {
356
+ return gitMirrorVolumeExists(volume);
357
+ }
358
+ async function gitMirrorVolumeExists(volume) {
359
+ try {
360
+ await runDocker(["volume", "inspect", volume]);
361
+ return true;
362
+ }
363
+ catch {
364
+ return false;
365
+ }
366
+ }
367
+ /**
368
+ * The mirror a container should mount, if any. Environments mount it
369
+ * read-write and fill it from their privileged bootstrap. One-shot runs only
370
+ * read it, and only when it already exists, so they never create an empty one
371
+ * (with `mirrors`, as far as the runner knows; see `VolumePresenceCache`).
372
+ */
373
+ export async function resolveGitMirrorMount(payload, localStateDir = LOCAL_STATE_DIR, mirrors) {
374
+ const canonical = canonicalGitHubRepositoryUrl(payload.repoUrl);
375
+ const volume = canonical ? mirrorVolumeName(canonical) : null;
376
+ if (!canonical || !volume)
377
+ return null;
378
+ const readOnly = isIsolatedRun(payload.envVars);
379
+ if (readOnly && !(await (mirrors ? mirrors.exists(volume) : gitMirrorVolumeExists(volume))))
380
+ return null;
381
+ try {
382
+ recordGitMirrorUse(volume, canonical, localStateDir);
383
+ }
384
+ catch (error) {
385
+ console.error(`[runner] could not record Git mirror use for ${volume}:`, error instanceof Error ? error.message : String(error));
386
+ }
387
+ return { volume, readOnly };
388
+ }
389
+ /**
390
+ * One-shot runs materialize their own checkout and must not inherit another
391
+ * run's workspace, so they get no persistent volume and mount the shared
392
+ * caches read-only.
393
+ */
394
+ function isIsolatedRun(envVars) {
395
+ const mode = envVars.TILLER_BOOTSTRAP_MODE;
396
+ return mode === "plan-writer"
397
+ || mode === "planner-run"
398
+ || mode === "env-review-run"
399
+ || mode === "github-env-publish"
400
+ || mode === "scm-land";
401
+ }
402
+ export const ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE = 75;
403
+ const ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE = 76;
404
+ /** The image label that says its land job knows the two modes of an app's Ship, `prepare` and `advance`. */
405
+ export const SCM_LAND_MODES_IMAGE_LABEL = "dev.tiller.scm-land-prepare-advance";
406
+ function resolveRunnerFeatures(flags) {
407
+ // `command-receipts`: the Hub session client acknowledges each runner
408
+ // command as soon as it arrives (runner-session.ts), before running it.
409
+ const features = ["git-mirror", "prepare-mirror-commit", "prepare-project", "idle-stop", "command-receipts"];
410
+ if (flags.stagedWorkspaces)
411
+ features.push("staged-workspaces");
412
+ if (flags.terminalLane)
413
+ features.push("terminal-lane");
414
+ // `gadget-job`: the image carries the gadget tools and the build-and-test job (gadget-job.ts).
415
+ if (flags.gadgetJob)
416
+ features.push("gadget-job");
417
+ // `gadget-map-back`: that job also maps an OS edit back onto the source, for a pull.
418
+ if (flags.gadgetJob && flags.gadgetMapBack)
419
+ features.push("gadget-map-back");
420
+ // `scm-land-modes`: the image's land job can stop before main (`prepare`) and move main later
421
+ // (`advance`), which the Ship of an app needs. An older image ignores the mode and lands main.
422
+ if (flags.scmLandModes)
423
+ features.push("scm-land-modes");
424
+ return features.sort();
425
+ }
426
+ function resolveRunnerCapabilities(image, flags = { stagedWorkspaces: false }) {
427
+ const reviewerIsolationProtocol = resolveHostReviewerIsolationProtocol(image, parseManagedLocalRunnerImageSourceId(image));
428
+ const features = resolveRunnerFeatures({
429
+ ...flags,
430
+ gadgetJob: Boolean(resolvePortableWorkspaceImage(image, GADGET_TOOLS_IMAGE_LABEL)),
431
+ gadgetMapBack: Boolean(resolvePortableWorkspaceImage(image, GADGET_MAP_BACK_IMAGE_LABEL)),
432
+ scmLandModes: Boolean(resolvePortableWorkspaceImage(image, SCM_LAND_MODES_IMAGE_LABEL)),
433
+ });
434
+ return {
435
+ runnerCommandProtocol: RUNNER_COMMAND_PROTOCOL,
436
+ codexRuntimeAuthProtocol: CODEX_RUNTIME_AUTH_PROTOCOL,
437
+ ...(reviewerIsolationProtocol ? { reviewerIsolationProtocol } : {}),
438
+ ...(resolveProjectEnvironmentImage(image) ? { projectEnvironmentProtocol: 1 } : {}),
439
+ ...(resolvePortableWorkspaceImage(image) ? { portableWorkspaceRestoreProtocol: 1 } : {}),
440
+ ...(resolvePortableWorkspaceImage(image, PORTABLE_NATIVE_LABEL) ? { portableNativeRestoreProtocol: 1 } : {}),
441
+ ...(features.length ? { features: Object.freeze(features) } : {}),
442
+ };
443
+ }
444
+ function redactError(error, envVars) {
445
+ return new Error(redactEnvValues(error instanceof Error ? error.message : String(error), envVars));
446
+ }
447
+ /**
448
+ * The folder name that holds one slug's OpenCode state: the lowercased slug,
449
+ * `+`, and 32 hex digits of the slug's SHA-256 (`runner-slug-folder.ts`).
450
+ * macOS disks ignore case by default, so the raw slug would give `Foo` and
451
+ * `foo` one folder while their containers and volumes stay apart; the hash of
452
+ * the exact slug keeps them apart here too.
453
+ */
454
+ export function opencodeStateFolderName(slug) {
455
+ return caseSafeSlugFolderName(slug);
456
+ }
457
+ /** The name runners from 2026-11-12 gave new folders: 16 hex digits instead of 32. */
458
+ export function legacyOpencodeStateFolderName(slug) {
459
+ return caseSafeSlugFolderName(slug, 16);
460
+ }
461
+ /**
462
+ * The folder (directly inside `<state dir>/opencode`) mounted read-write into
463
+ * an OpenCode environment and deleted when it is destroyed.
464
+ *
465
+ * Earlier runners named the folder after the raw slug, and then after the
466
+ * slug with 16 hex digits of its hash. Such a folder is used in place when a
467
+ * directory entry with exactly that name exists, in that order: the entry's
468
+ * own name is compared, because on a case-insensitive disk `Foo` would also
469
+ * "exist" when only `foo` does. It is not moved: a running environment may
470
+ * have it mounted. Every other slug gets `opencodeStateFolderName`.
471
+ *
472
+ * The API refuses slugs that are not plain names (`runner-slug.ts`); as a
473
+ * second safeguard this refuses any path not directly inside the `opencode`
474
+ * folder.
475
+ */
476
+ export function resolveHostOpencodeStatePath(slug, localStateDir = LOCAL_STATE_DIR) {
477
+ const root = resolve(localStateDir, "opencode");
478
+ const refuse = () => new RunnerSlugError(`Refusing an OpenCode state folder outside ${root}.`);
479
+ if (!isValidRunnerSlug(slug))
480
+ throw refuse();
481
+ const folder = findExactDirectoryEntry(root, [slug, legacyOpencodeStateFolderName(slug)])
482
+ ?? opencodeStateFolderName(slug);
483
+ const path = resolve(root, folder);
484
+ if (dirname(path) !== root || basename(path) !== folder)
485
+ throw refuse();
486
+ return path;
487
+ }
488
+ export function resolveOpencodeMount(payload, localStateDir = LOCAL_STATE_DIR) {
489
+ if (payload.envVars.TILLER_HARNESS !== "opencode"
490
+ || payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE !== undefined
491
+ || payload.envVars.TILLER_PORTABLE_NATIVE_RESTORE !== undefined) {
492
+ return null;
493
+ }
494
+ const sourcePath = resolveHostOpencodeStatePath(payload.slug, localStateDir);
495
+ mkdirSync(sourcePath, { recursive: true, mode: 0o700 });
496
+ return {
497
+ sourcePath,
498
+ targetPath: HOST_OPENCODE_STATE_MOUNT_PATH,
499
+ };
500
+ }
501
+ export function removeLocalOpencodeState(slug, localStateDir = LOCAL_STATE_DIR) {
502
+ const sourcePath = resolveHostOpencodeStatePath(slug, localStateDir);
503
+ if (!existsSync(sourcePath)) {
504
+ return false;
505
+ }
506
+ rmSync(sourcePath, { recursive: true, force: true });
507
+ return true;
508
+ }
509
+ function isPermissionError(error) {
510
+ const code = error?.code;
511
+ return code === "EACCES" || code === "EPERM";
512
+ }
513
+ async function removeLocalOpencodeStateForRunner(slug, image) {
514
+ const sourcePath = resolveHostOpencodeStatePath(slug);
515
+ if (!existsSync(sourcePath)) {
516
+ return false;
517
+ }
518
+ try {
519
+ rmSync(sourcePath, { recursive: true, force: true });
520
+ return true;
521
+ }
522
+ catch (error) {
523
+ if (!isPermissionError(error)) {
524
+ throw error;
525
+ }
526
+ }
527
+ await runDocker([
528
+ "run",
529
+ "--rm",
530
+ "--entrypoint",
531
+ "sh",
532
+ "-v",
533
+ `${dirname(sourcePath)}:/tiller-opencode-state`,
534
+ image,
535
+ "-lc",
536
+ 'rm -rf -- "$1"',
537
+ "sh",
538
+ `/tiller-opencode-state/${basename(sourcePath)}`,
539
+ ]);
540
+ return true;
541
+ }
542
+ export function buildContainerEnvVars(payload, options) {
543
+ const packageCacheEnv = {};
544
+ for (const mount of PACKAGE_CACHE_MOUNTS) {
545
+ Object.assign(packageCacheEnv, mount.env);
546
+ }
547
+ return {
548
+ // Point the package managers at the shared caches first so a repository's
549
+ // own values, and the Hub's, still win.
550
+ ...packageCacheEnv,
551
+ ...payload.envVars,
552
+ ...(options?.gitMirror ? { [GIT_MIRROR_ENV_VAR]: GIT_MIRROR_CONTAINER_PATH } : {}),
553
+ TILLER_STOP_CONTROL_SECRET: randomBytes(32).toString("hex"),
554
+ // A session's harness serves the local terminal lane on this port, on the
555
+ // container's own loopback; the runner reaches it through `docker exec`.
556
+ // Isolated runs (planners, reviews) have no interactive terminal.
557
+ ...(options?.terminalLane && !isIsolatedRun(payload.envVars)
558
+ ? {
559
+ TILLER_TERMINAL_LANE_PORT: String(TERMINAL_LANE_CONTAINER_PORT),
560
+ TILLER_TERMINAL_LANE_HOST: TERMINAL_LANE_CONTAINER_HOST,
561
+ TILLER_TERMINAL_LANE_MACHINE_ID: options.terminalLane.machineId,
562
+ }
563
+ : {}),
564
+ ...(payload.operationId?.trim() ? { [START_OP_ID_ENV_VAR]: payload.operationId.trim() } : {}),
565
+ ...(Number.isSafeInteger(payload.commandGeneration) && (payload.commandGeneration ?? 0) > 0
566
+ ? {
567
+ [HOST_COMMAND_GENERATION_ENV_VAR]: String(payload.commandGeneration),
568
+ [HOST_COMMAND_FENCE_REQUIRED_ENV_VAR]: "1",
569
+ }
570
+ : {}),
571
+ };
572
+ }
573
+ export function buildDockerEnvFileContent(envVars) {
574
+ validateProjectEnvironmentDelivery(envVars);
575
+ return Object.entries(envVars).map(([key, value]) => {
576
+ if (value.includes("\0") || value.includes("\n") || value.includes("\r")) {
577
+ throw new Error(`${key} cannot be written to a Docker env file.`);
578
+ }
579
+ return `${key}=${value}`;
580
+ }).join("\n") + "\n";
581
+ }
582
+ export function buildHostLaunchFileContent(envVars) {
583
+ buildDockerEnvFileContent(envVars);
584
+ // The entrypoint sources this file; Docker's literal env-file syntax is not shell syntax.
585
+ // Stop clients launched by docker exec inherit the container's original credential.
586
+ // Retain that credential across warm starts instead of overriding only the server.
587
+ return Object.entries(envVars).filter(([key]) => key !== "TILLER_STOP_CONTROL_SECRET").map(([key, value]) => {
588
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
589
+ throw new Error("Invalid host launch environment key.");
590
+ return `${key}='${value.replaceAll("'", "'\\''")}'`;
591
+ }).join("\n") + "\n";
592
+ }
593
+ function usesCodexReviewerSandbox(envVars) {
594
+ return ["planner-run", "env-review-run"].includes(envVars.TILLER_BOOTSTRAP_MODE) && envVars.TILLER_HARNESS === "codex";
595
+ }
596
+ export function createDockerEnvFile(envVars) {
597
+ let dirPath = null;
598
+ try {
599
+ dirPath = mkdtempSync(join(tmpdir(), "tiller-env-"));
600
+ chmodSync(dirPath, 0o700);
601
+ const filePath = join(dirPath, "env");
602
+ writeFileSync(filePath, buildDockerEnvFileContent(envVars), { encoding: "utf8", mode: 0o600 });
603
+ chmodSync(filePath, 0o600);
604
+ if (usesCodexReviewerSandbox(envVars)) {
605
+ // The parser helper drops DAC_OVERRIDE; this policy is public data inside
606
+ // the private temporary directory, so it can be world-readable.
607
+ writeFileSync(join(dirPath, "codex-reviewer.apparmor"), codexReviewerAppArmor.profile, { encoding: "utf8", mode: 0o644 });
608
+ writeFileSync(join(dirPath, "codex-reviewer-seccomp.json"), JSON.stringify(codexReviewerSeccomp), {
609
+ encoding: "utf8",
610
+ mode: 0o600,
611
+ });
612
+ }
613
+ const cleanupPath = dirPath;
614
+ return {
615
+ filePath,
616
+ cleanup: () => rmSync(cleanupPath, { recursive: true, force: true }),
617
+ };
618
+ }
619
+ catch (error) {
620
+ if (dirPath) {
621
+ rmSync(dirPath, { recursive: true, force: true });
622
+ }
623
+ throw error;
624
+ }
625
+ }
626
+ export function buildContainerRunArgs(payload, image, envFilePath, options) {
627
+ const name = containerName(payload.slug);
628
+ // `create`, not `run -d`: the container id must be known before any process
629
+ // runs so the launch record can name it and the entrypoint can verify that
630
+ // the record is its own.
631
+ const args = [
632
+ "create",
633
+ ...(image.startsWith("sha256:") ? ["--pull=never"] : []),
634
+ "--name",
635
+ name,
636
+ "--label",
637
+ `${dockerLabel("slug")}=${payload.slug}`,
638
+ "--add-host",
639
+ "host.docker.internal:host-gateway",
640
+ ];
641
+ if (options?.configHash) {
642
+ args.push("--label", `${HOST_CONFIG_HASH_LABEL}=${options.configHash}`);
643
+ }
644
+ if (payload.commandGeneration && options?.fenceDirectory) {
645
+ args.push("--label", `${HOST_COMMAND_GENERATION_LABEL}=${payload.commandGeneration}`, "--label", `${HOST_OPERATION_ID_LABEL}=${payload.operationId}`, "-v", `${options.fenceDirectory}:${HOST_COMMAND_FENCE_CONTAINER_PATH}:ro`);
646
+ }
647
+ const isolated = isIsolatedRun(payload.envVars);
648
+ if (isolated) {
649
+ // Left out of the environment count and the limit (`machine-capacity.ts`).
650
+ args.push("--label", `${dockerLabel(ISOLATED_RUN_LABEL_KEY)}=true`);
651
+ }
652
+ else {
653
+ // The workspace outlives the container so a recreate keeps the checkout and
654
+ // its installed dependencies. The Hub remains its durable owner; this is a
655
+ // cache of the Hub's copy.
656
+ // Archive recovery needs an empty destination. Leave the retained volume
657
+ // untouched and restore into the fresh container's writable filesystem.
658
+ if (payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined)
659
+ args.push("-v", `${options?.workspaceVolume ?? workspaceVolumeName(payload.slug)}:${WORKSPACE_CONTAINER_PATH}`);
660
+ args.push("-v", `${RESTORE_CACHE_VOLUME_PREFIX}${payload.slug}:/var/lib/tiller-restore-cache`);
661
+ // No port is published: the terminal lane is reached through `docker exec`
662
+ // (`terminal-lane.ts`), so nothing on this machine but the runner reaches it.
663
+ }
664
+ for (const mount of PACKAGE_CACHE_MOUNTS) {
665
+ args.push("-v", `${mount.volume}:${mount.target}${isolated ? ":ro" : ""}`);
666
+ }
667
+ if (options?.gitMirror) {
668
+ args.push("-v", `${options.gitMirror.volume}:${GIT_MIRROR_CONTAINER_PATH}${options.gitMirror.readOnly ? ":ro" : ""}`);
669
+ }
670
+ if (options?.harnessOverlayDir) {
671
+ args.push("-v", `${options.harnessOverlayDir}:${HARNESS_DIST_CONTAINER_PATH}:ro`);
672
+ }
673
+ const opencodeMount = resolveOpencodeMount(payload);
674
+ if (opencodeMount) {
675
+ args.push("-v", `${opencodeMount.sourcePath}:${opencodeMount.targetPath}`);
676
+ }
677
+ if (usesCodexReviewerSandbox(payload.envVars)) {
678
+ args.push("--security-opt", `seccomp=${join(dirname(envFilePath), "codex-reviewer-seccomp.json")}`);
679
+ if (options?.codexReviewerAppArmorProfile) {
680
+ args.push("--security-opt", `apparmor=${options.codexReviewerAppArmorProfile}`);
681
+ }
682
+ }
683
+ args.push("--env-file", envFilePath, image);
684
+ return args;
685
+ }
686
+ /**
687
+ * Identifies the shape of a container: everything `docker create` was given
688
+ * except the per-start values, which a reused container receives through its
689
+ * launch file instead. Paths under the temporary env directory are replaced by
690
+ * a sentinel because that directory is new on every start.
691
+ */
692
+ export function computeContainerConfigHash(payload, image, envFilePath, options) {
693
+ const args = buildContainerRunArgs(payload, image, envFilePath, options);
694
+ const envDirectory = dirname(envFilePath);
695
+ const normalized = args.map((arg) => {
696
+ // The generation and operation id identify a start, not a container shape.
697
+ // A reused container keeps the labels of the start that created it, so they
698
+ // must not take part in the comparison.
699
+ if (arg.startsWith(`${HOST_COMMAND_GENERATION_LABEL}=`)) {
700
+ return `${HOST_COMMAND_GENERATION_LABEL}=<generation>`;
701
+ }
702
+ if (arg.startsWith(`${HOST_OPERATION_ID_LABEL}=`)) {
703
+ return `${HOST_OPERATION_ID_LABEL}=<operation-id>`;
704
+ }
705
+ return arg.split(envDirectory).join("<env-dir>");
706
+ });
707
+ return createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
708
+ }
709
+ function containerConfigHash(info) {
710
+ return info?.Config?.Labels?.[HOST_CONFIG_HASH_LABEL]?.trim() || null;
711
+ }
712
+ async function resolveImageId(image) {
713
+ try {
714
+ const output = await runDocker(["image", "inspect", image, "--format", "{{.Id}}"]);
715
+ return output.trim() || null;
716
+ }
717
+ catch {
718
+ return null;
719
+ }
720
+ }
721
+ // Install only Tiller's named container policy; never change host-wide userns
722
+ // settings. The trusted parser helper has MAC_ADMIN solely to load this policy.
723
+ export function buildReviewerAppArmorInstallArgs(image, profilePath) {
724
+ return [
725
+ "run", "--rm", "--network", "none", "--cap-drop", "ALL", "--cap-add", "MAC_ADMIN",
726
+ "--security-opt", "apparmor=unconfined",
727
+ "--mount", "type=bind,src=/sys/kernel/security,dst=/sys/kernel/security",
728
+ "--mount", `type=bind,src=${profilePath},dst=/tmp/tiller-reviewer.apparmor,readonly`,
729
+ "--entrypoint", "/usr/sbin/apparmor_parser", image,
730
+ "--replace", "--skip-cache", "/tmp/tiller-reviewer.apparmor",
731
+ ];
732
+ }
733
+ const reviewerAppArmorInstalls = new Map();
734
+ /** `docker info` once a minute at most; concurrent reviewer creates share one call. */
735
+ export const dockerSecurityOptions = new TtlSingleFlight(async () => {
736
+ const securityOptions = JSON.parse(await runDocker(["info", "--format", "{{json .SecurityOptions}}"]));
737
+ if (!Array.isArray(securityOptions))
738
+ throw new Error("Docker did not report its security options.");
739
+ return securityOptions;
740
+ }, DOCKER_INFO_TTL_MS);
741
+ export async function prepareReviewerAppArmor(image, envFilePath) {
742
+ const securityOptions = await dockerSecurityOptions.get();
743
+ if (!securityOptions.some((option) => typeof option === "string" && option.startsWith("name=apparmor")))
744
+ return undefined;
745
+ let install = reviewerAppArmorInstalls.get(image);
746
+ if (!install) {
747
+ install = runDocker(buildReviewerAppArmorInstallArgs(image, join(dirname(envFilePath), "codex-reviewer.apparmor")))
748
+ .then(() => undefined)
749
+ .catch((error) => {
750
+ reviewerAppArmorInstalls.delete(image);
751
+ throw new Error(`Could not install the Codex reviewer AppArmor profile: ${error instanceof Error ? error.message : String(error)}`);
752
+ });
753
+ reviewerAppArmorInstalls.set(image, install);
754
+ }
755
+ await install;
756
+ return codexReviewerAppArmor.name;
757
+ }
758
+ /**
759
+ * Times the steps of one create or start: `mark(step)` records the time since
760
+ * the previous mark, `line()` renders them for the runner log.
761
+ */
762
+ export function createStepTimer(now = () => performance.now()) {
763
+ const startedAt = now();
764
+ let last = startedAt;
765
+ const steps = [];
766
+ return {
767
+ mark(step) {
768
+ const at = now();
769
+ steps.push([step, at - last]);
770
+ last = at;
771
+ },
772
+ line() {
773
+ const total = Math.round(last - startedAt);
774
+ return `${total} ms (${steps.map(([step, ms]) => `${step} ${Math.round(ms)}`).join(", ")})`;
775
+ },
776
+ };
777
+ }
778
+ function commandEnvelopeFromPayload(payload) {
779
+ return {
780
+ ...(payload.commandGeneration !== undefined
781
+ ? { commandGeneration: payload.commandGeneration }
782
+ : {}),
783
+ ...(payload.operationId !== undefined ? { operationId: payload.operationId } : {}),
784
+ ...(payload.desiredState !== undefined
785
+ ? { desiredState: payload.desiredState }
786
+ : {}),
787
+ };
788
+ }
789
+ /**
790
+ * Attributes a container to the command generation that launched it. The
791
+ * launch record, not the immutable Docker labels, is authoritative: labels
792
+ * cannot be rewritten for a reused container, and a record that names a
793
+ * different container attributes nothing.
794
+ */
795
+ function launchedGeneration(info, launched) {
796
+ if (!info?.Id || !launched)
797
+ return null;
798
+ return launched.containerId === info.Id ? launched.commandGeneration : null;
799
+ }
800
+ function launchedOperationId(info, launched) {
801
+ if (!info?.Id || !launched)
802
+ return null;
803
+ return launched.containerId === info.Id ? launched.operationId : null;
804
+ }
805
+ /**
806
+ * Exits that prove the container never reached the workspace or the harness,
807
+ * so replacing or reusing it loses nothing. 75 is the fence refusing a
808
+ * superseded launch before touching the workspace; 76 is a later pre-harness
809
+ * failure; 77 is a Stop that claimed the boot before the harness was let go
810
+ * and had the Hub accept it; 79 is a Stop the entrypoint saved itself after
811
+ * stop-control could not, also accepted by the Hub. Without 75 here, the ordinary "stop soon after
812
+ * start" sequence would leave an environment permanently unable to start.
813
+ * 77 is here as well as in the Stop's own proof: the container reports the
814
+ * Stop to the Hub before Docker returns, so a restart can supersede the Stop
815
+ * command before it records anything.
816
+ */
817
+ function safeReplacementProof(info, launched) {
818
+ const exitCode = info?.State?.ExitCode;
819
+ if (exitCode !== ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE
820
+ && exitCode !== ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE
821
+ && exitCode !== ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE
822
+ && exitCode !== ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE) {
823
+ return null;
824
+ }
825
+ const commandGeneration = launchedGeneration(info, launched);
826
+ const operationId = launchedOperationId(info, launched);
827
+ return commandGeneration && operationId
828
+ ? { commandGeneration, operationId, exitCode }
829
+ : null;
830
+ }
831
+ function failedBeforeHarnessProof(info, launched) {
832
+ if (info?.State?.ExitCode !== ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE)
833
+ return null;
834
+ const commandGeneration = launchedGeneration(info, launched);
835
+ const operationId = launchedOperationId(info, launched);
836
+ return commandGeneration && operationId ? { commandGeneration, operationId } : null;
837
+ }
838
+ /**
839
+ * The Stop that durably saved this exact stopped container, when nothing has
840
+ * started it since. It is the proof `createContainer` accepts to reuse or
841
+ * replace the container, so it holds however the commands after that Stop
842
+ * ended (for example a Start that never reached this runner, or one refused
843
+ * before it touched Docker).
844
+ */
845
+ function durableStopProof(info, proof) {
846
+ if (!info?.Id || !proof?.containerId || proof.containerId !== info.Id)
847
+ return null;
848
+ if (!["stopped", "exited", "dead"].includes(getStatus(info)))
849
+ return null;
850
+ if (!startedBefore(info, proof.recordedAt))
851
+ return null;
852
+ return { stopOperationId: proof.stopOperationId, stopCommandGeneration: proof.stopCommandGeneration };
853
+ }
854
+ async function startContainer(slug) {
855
+ await runDocker(["start", containerName(slug)]);
856
+ }
857
+ function harnessSourceOf(info) {
858
+ return info.Mounts?.some((mount) => mount.Destination === HARNESS_DIST_CONTAINER_PATH) ? "overlay" : "image";
859
+ }
860
+ /** Docker reports "0001-01-01T00:00:00Z" for a time that has not happened. */
861
+ function dockerTimestamp(value) {
862
+ const parsed = Date.parse(value ?? "");
863
+ return Number.isFinite(parsed) && parsed > 0 ? value : undefined;
864
+ }
865
+ /** Create and start times from an inspect, as the startup trace reports them. */
866
+ function containerTimes(info) {
867
+ const createdAt = dockerTimestamp(info?.Created);
868
+ const startedAt = hasRun(info) ? dockerTimestamp(info?.State?.StartedAt) : undefined;
869
+ return {
870
+ ...(createdAt ? { containerCreatedAt: createdAt } : {}),
871
+ ...(startedAt ? { containerStartedAt: startedAt } : {}),
872
+ };
873
+ }
874
+ async function createContainer(payload, image, options) {
875
+ if (!payload.slug || !payload.repoUrl || !payload.envVars || typeof payload.envVars !== "object") {
876
+ throw new Error("slug, repoUrl, and envVars are required");
877
+ }
878
+ validateProjectEnvironmentDelivery(payload.envVars);
879
+ const mark = options.mark ?? (() => { });
880
+ const expectedGeneration = payload.commandGeneration ?? null;
881
+ const gitMirror = options.resolveGitMirror ? await options.resolveGitMirror(payload) : null;
882
+ mark("mirror");
883
+ const containerEnvVars = buildContainerEnvVars(payload, { gitMirror, terminalLane: options.terminalLane });
884
+ const envFileContents = buildHostLaunchFileContent(containerEnvVars);
885
+ // Resolved before the reuse decision because both the image comparison and
886
+ // the config hash are taken against the image this start would actually use.
887
+ const launchImage = requirePortableWorkspaceImage(requireProjectEnvironmentImage(image, payload.envVars), payload.envVars);
888
+ mark("image-check");
889
+ const envFile = createDockerEnvFile(containerEnvVars);
890
+ mark("env-file");
891
+ try {
892
+ options.guard();
893
+ const existing = await inspectContainer(payload.slug);
894
+ mark("inspect");
895
+ options.guard();
896
+ // Resolved before the hash so the `/workspace` argument is the same on
897
+ // every create or recreate of this slug.
898
+ const workspace = isIsolatedRun(payload.envVars) || !options.resolveWorkspaceVolume
899
+ ? null
900
+ : await options.resolveWorkspaceVolume(payload, { containerExists: existing !== null });
901
+ mark("workspace-volume");
902
+ const harnessOverlayDir = options.resolveHarnessOverlay ? await options.resolveHarnessOverlay() : null;
903
+ const withWorkspaceKind = (outcome, info) => ({
904
+ ...outcome,
905
+ ...(workspace
906
+ ? {
907
+ workspaceVolumeKind: workspace.kind,
908
+ ...(workspace.stagedDependencies ? { stagedDependencies: workspace.stagedDependencies } : {}),
909
+ }
910
+ : {}),
911
+ // Read from the container, so a reused one reports what it really mounts.
912
+ ...(info ? { harnessSource: harnessSourceOf(info) } : {}),
913
+ });
914
+ options.guard();
915
+ // The AppArmor profile is loaded into the kernel per create and does not
916
+ // survive a host reboot, so a reused sandboxed container needs it too.
917
+ const codexReviewerAppArmorProfile = usesCodexReviewerSandbox(containerEnvVars)
918
+ ? await prepareReviewerAppArmor(launchImage, envFile.filePath)
919
+ : undefined;
920
+ options.guard();
921
+ const runArgsOptions = {
922
+ ...(codexReviewerAppArmorProfile ? { codexReviewerAppArmorProfile } : {}),
923
+ fenceDirectory: options.fenceDirectory,
924
+ ...(gitMirror ? { gitMirror } : {}),
925
+ ...(workspace ? { workspaceVolume: workspace.volume } : {}),
926
+ ...(harnessOverlayDir ? { harnessOverlayDir } : {}),
927
+ };
928
+ const configHash = computeContainerConfigHash(payload, launchImage, envFile.filePath, runArgsOptions);
929
+ mark("config-hash");
930
+ // Publishing the launch and starting are one step: the record names the
931
+ // container before any process in it runs, and it is written under the
932
+ // per-slug queue after asserting the fence, so a command accepted on the
933
+ // HTTP path cannot claim a container it did not start.
934
+ const launchAndStart = async (containerId) => {
935
+ options.guard();
936
+ options.recordLaunch(containerId, envFileContents);
937
+ mark("record-launch");
938
+ options.guard();
939
+ await startContainer(payload.slug);
940
+ mark("docker-start");
941
+ };
942
+ const createFresh = async () => {
943
+ options.guard();
944
+ const args = buildContainerRunArgs(payload, launchImage, envFile.filePath, {
945
+ ...runArgsOptions,
946
+ configHash,
947
+ });
948
+ const containerId = (await runDocker(args)).trim();
949
+ const createdAt = new Date().toISOString();
950
+ mark("docker-create");
951
+ if (!/^[0-9a-f]{64}$/.test(containerId)) {
952
+ throw new Error("Docker did not report a container id for the created runner.");
953
+ }
954
+ await launchAndStart(containerId);
955
+ const startedAt = new Date().toISOString();
956
+ // If a newer Stop won while Docker was starting, leave the live container
957
+ // in place. The per-slug command queue will run that fenced Stop next so
958
+ // it can complete the normal durable workspace-save flow.
959
+ options.guard();
960
+ // No inspect after the start (it took up to 1.4 s live): everything the
961
+ // answer carries is already known for a container this create made.
962
+ // `docker start` succeeded, so it was started; a container that exits at
963
+ // once is reported by its entrypoint and the Hub's start window, as
964
+ // before (nothing read the inspected status). The trace times are the
965
+ // runner's, taken as `docker create` and `docker start` returned.
966
+ return {
967
+ runnerId: containerName(payload.slug),
968
+ status: "running",
969
+ containerReuse: "created",
970
+ containerCreatedAt: createdAt,
971
+ containerStartedAt: startedAt,
972
+ ...(workspace
973
+ ? {
974
+ workspaceVolumeKind: workspace.kind,
975
+ ...(workspace.stagedDependencies ? { stagedDependencies: workspace.stagedDependencies } : {}),
976
+ }
977
+ : {}),
978
+ harnessSource: harnessOverlayDir ? "overlay" : "image",
979
+ };
980
+ };
981
+ if (existing) {
982
+ const launched = options.readLaunched();
983
+ const existingGeneration = launchedGeneration(existing, launched);
984
+ const status = getStatus(existing);
985
+ if (status === "created") {
986
+ // A runner crash between `docker create` and `docker start`. An exact
987
+ // retry of the command that created it starts it; anything else removes
988
+ // it, because nothing ever ran.
989
+ const matchesThisCommand = Boolean(expectedGeneration !== null
990
+ && launched
991
+ && existing.Id
992
+ && launched.containerId === existing.Id
993
+ && launched.commandGeneration === expectedGeneration
994
+ && launched.operationId === payload.operationId);
995
+ if (matchesThisCommand) {
996
+ options.guard();
997
+ await startContainer(payload.slug);
998
+ options.guard();
999
+ const info = await inspectContainer(payload.slug);
1000
+ return withWorkspaceKind({
1001
+ runnerId: containerName(payload.slug),
1002
+ status: getStatus(info),
1003
+ containerReuse: "reused",
1004
+ ...containerTimes(info),
1005
+ }, info);
1006
+ }
1007
+ await removeStoppedContainer(payload.slug);
1008
+ options.guard();
1009
+ return await createFresh();
1010
+ }
1011
+ if (status === "running") {
1012
+ if (expectedGeneration !== null && existingGeneration === expectedGeneration) {
1013
+ requireProjectEnvironmentContainer(existing, payload.envVars);
1014
+ requirePortableWorkspaceContainer(existing, payload.envVars);
1015
+ return withWorkspaceKind({
1016
+ runnerId: containerName(payload.slug),
1017
+ status: "running",
1018
+ containerReuse: "reused",
1019
+ ...containerTimes(existing),
1020
+ }, existing);
1021
+ }
1022
+ throw new RunnerCommandFenceError(`A live runner already exists for ${payload.slug}; refusing to replace it.`, "runner_command_conflict");
1023
+ }
1024
+ const persistenceProof = options.readWorkspaceSyncedStop();
1025
+ const replacement = safeReplacementProof(existing, launched);
1026
+ const acknowledgedPersistence = Boolean(expectedGeneration !== null
1027
+ && persistenceProof
1028
+ && persistenceProof.stopCommandGeneration < expectedGeneration
1029
+ && (
1030
+ // A durable stop of this container is proof for this container,
1031
+ // whatever generation the launch record now names.
1032
+ (persistenceProof.containerId !== undefined
1033
+ && existing.Id !== undefined
1034
+ && persistenceProof.containerId === existing.Id
1035
+ && startedBefore(existing, persistenceProof.recordedAt))
1036
+ || (existingGeneration !== null
1037
+ && persistenceProof.runnerCommandGeneration === existingGeneration)));
1038
+ const safePreHarnessFailure = Boolean(expectedGeneration !== null
1039
+ && replacement
1040
+ && replacement.commandGeneration < expectedGeneration);
1041
+ if (!acknowledgedPersistence && !safePreHarnessFailure) {
1042
+ throw new RunnerCommandFenceError(`The stopped runner for ${payload.slug} has no matching workspace persistence acknowledgement.`, "runner_command_conflict");
1043
+ }
1044
+ options.guard();
1045
+ const pinnedImageId = await resolveImageId(launchImage);
1046
+ options.guard();
1047
+ let reusable = Boolean(
1048
+ // A previous recovery may already have materialized files, even when
1049
+ // native restoration failed. A new recovery always starts empty.
1050
+ payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined
1051
+ && existing.Id
1052
+ && expectedGeneration !== null
1053
+ && existingGeneration !== null
1054
+ && existingGeneration < expectedGeneration
1055
+ && pinnedImageId !== null
1056
+ && existing.Image === pinnedImageId
1057
+ && containerConfigHash(existing) === configHash
1058
+ && (hasRun(existing) || replacement !== null));
1059
+ if (reusable) {
1060
+ try {
1061
+ // A reused container keeps the environment it was created with, so a
1062
+ // project-environment or saved-workspace selection that no longer
1063
+ // matches has to be answered with a fresh container rather than a
1064
+ // failed start.
1065
+ requireProjectEnvironmentContainer(existing, payload.envVars);
1066
+ requirePortableWorkspaceContainer(existing, payload.envVars);
1067
+ }
1068
+ catch {
1069
+ reusable = false;
1070
+ }
1071
+ }
1072
+ if (reusable) {
1073
+ try {
1074
+ await launchAndStart(existing.Id);
1075
+ options.guard();
1076
+ const info = await inspectContainer(payload.slug);
1077
+ return withWorkspaceKind({
1078
+ runnerId: containerName(payload.slug),
1079
+ status: getStatus(info),
1080
+ containerReuse: "reused",
1081
+ ...containerTimes(info),
1082
+ }, info);
1083
+ }
1084
+ catch (error) {
1085
+ if (error instanceof RunnerCommandFenceError)
1086
+ throw error;
1087
+ // Eligibility was verified a moment ago, so a failed start is this
1088
+ // container's problem. Replace it inside the same execution, while
1089
+ // the proof that made it replaceable still holds.
1090
+ console.error(`[runner] reuse of the stopped runner for ${payload.slug} failed; recreating:`, redactEnvValues(error instanceof Error ? error.message : String(error), containerEnvVars));
1091
+ await forceRemoveContainer(payload.slug);
1092
+ options.guard();
1093
+ return await createFresh();
1094
+ }
1095
+ }
1096
+ await removeStoppedContainer(payload.slug);
1097
+ options.guard();
1098
+ }
1099
+ return await createFresh();
1100
+ }
1101
+ catch (error) {
1102
+ if (error instanceof RunnerCommandFenceError)
1103
+ throw error;
1104
+ throw redactError(error, containerEnvVars);
1105
+ }
1106
+ finally {
1107
+ envFile.cleanup();
1108
+ }
1109
+ }
1110
+ async function inspectAppliedRunningCommand(record, fenceStore, envVars) {
1111
+ fenceStore.assertCurrent(record);
1112
+ const info = await inspectContainer(record.slug);
1113
+ fenceStore.assertCurrent(record);
1114
+ const launched = fenceStore.readLaunched(record.slug);
1115
+ if (!info || launchedGeneration(info, launched) !== record.commandGeneration) {
1116
+ throw new RunnerCommandFenceError(`Runner command generation ${record.commandGeneration} was applied, but its exact runner is unavailable.`, "runner_command_conflict");
1117
+ }
1118
+ requireProjectEnvironmentContainer(info, envVars);
1119
+ requirePortableWorkspaceContainer(info, envVars);
1120
+ return {
1121
+ runnerId: containerName(record.slug),
1122
+ status: getStatus(info),
1123
+ containerReuse: "reused",
1124
+ ...containerTimes(info),
1125
+ };
1126
+ }
1127
+ async function ensureStarted(slug, payload, image, options) {
1128
+ if (payload?.repoUrl && payload?.envVars && typeof payload.envVars === "object") {
1129
+ return createContainer({
1130
+ slug,
1131
+ repoUrl: payload.repoUrl,
1132
+ envVars: payload.envVars,
1133
+ ...(payload.commandGeneration ? { commandGeneration: payload.commandGeneration } : {}),
1134
+ ...(payload.operationId ? { operationId: payload.operationId } : {}),
1135
+ ...(payload.desiredState ? { desiredState: payload.desiredState } : {}),
1136
+ }, image, options);
1137
+ }
1138
+ options.guard();
1139
+ const existing = await inspectContainer(slug);
1140
+ options.guard();
1141
+ if (!existing) {
1142
+ throw new Error(`Container ${containerName(slug)} does not exist`);
1143
+ }
1144
+ requireProjectEnvironmentContainer(existing, payload?.envVars);
1145
+ requirePortableWorkspaceContainer(existing, payload?.envVars);
1146
+ options.guard();
1147
+ return {
1148
+ runnerId: containerName(slug),
1149
+ status: getStatus(existing),
1150
+ containerReuse: "reused",
1151
+ ...containerTimes(existing),
1152
+ };
1153
+ }
1154
+ function quoteShellString(value) {
1155
+ return `'${value.replace(/'/g, `'\"'\"'`)}'`;
1156
+ }
1157
+ function buildStopClientScript(args) {
1158
+ return (`set -eu; ` +
1159
+ `unset NODE_OPTIONS NODE_PATH GLIBC_TUNABLES GCONV_PATH LOCPATH NLSPATH; ` +
1160
+ `for name in "\${!LD_@}"; do unset "$name"; done; ` +
1161
+ `exec /usr/local/bin/tiller-controller-node /stop-control-client.mjs ${args.map((arg) => arg.startsWith("--") ? arg : quoteShellString(arg)).join(" ")}`);
1162
+ }
1163
+ export function buildPrepareDurableStopScript(stopOpId, options = {}) {
1164
+ const trimmedStopOpId = stopOpId?.trim();
1165
+ if (!trimmedStopOpId)
1166
+ throw new Error("A lifecycle Stop operation ID is required");
1167
+ const idleClaimId = options.idleClaimId?.trim();
1168
+ return buildStopClientScript([
1169
+ ...(options.fenceOnly ? ["--fence"] : []),
1170
+ ...(idleClaimId ? ["--idle-claim", idleClaimId] : []),
1171
+ trimmedStopOpId,
1172
+ ]);
1173
+ }
1174
+ /** stop-control-client: the idle Stop's claim is no longer held; nothing was fenced. */
1175
+ export const STOP_CLIENT_IDLE_CLAIM_SUPERSEDED_EXIT_CODE = 13;
1176
+ /**
1177
+ * How long a host idle claim may fence the harness's input before a Stop
1178
+ * turns it into a fence. The Hub asks for the Stop right after the claim; a
1179
+ * Hub that never does must not leave the terminal refusing keystrokes.
1180
+ */
1181
+ export const HOST_IDLE_CLAIM_TTL_MS = 120_000;
1182
+ function normalizeIdleStopPreparation(value, idleTimeoutMs) {
1183
+ const record = value && typeof value === "object" ? value : {};
1184
+ const claimId = typeof record.claimId === "string" && record.claimId.trim() ? record.claimId.trim() : undefined;
1185
+ const eligible = record.eligible === true && Boolean(claimId);
1186
+ const remaining = typeof record.remainingIdleMs === "number" && Number.isFinite(record.remainingIdleMs)
1187
+ ? Math.max(0, Math.ceil(record.remainingIdleMs))
1188
+ : idleTimeoutMs;
1189
+ return {
1190
+ eligible,
1191
+ reason: typeof record.reason === "string" && record.reason ? record.reason : eligible ? "eligible" : "not_eligible",
1192
+ remainingIdleMs: eligible ? 0 : remaining,
1193
+ ...(eligible ? { claimId } : {}),
1194
+ ...(typeof record.idleSince === "number" && Number.isFinite(record.idleSince) ? { idleSince: record.idleSince } : {}),
1195
+ ...(typeof record.error === "string" && record.error ? { error: record.error.slice(0, 500) } : {}),
1196
+ };
1197
+ }
1198
+ /**
1199
+ * Host idle stop, phase one: the container's stop-control claims the idle
1200
+ * stop (terminal output counts as activity) and saves the workspace. It
1201
+ * changes nothing else; a claim that is not turned into a Stop expires.
1202
+ */
1203
+ export async function prepareIdleStopInContainer(slug, idleTimeoutMs, claimTtlMs = HOST_IDLE_CLAIM_TTL_MS) {
1204
+ const output = await runDocker([
1205
+ "exec", containerName(slug), "bash", "-c",
1206
+ buildStopClientScript(["--idle-prepare", String(Math.floor(idleTimeoutMs)), String(Math.floor(claimTtlMs))]),
1207
+ ]);
1208
+ const line = output.trim().split("\n").filter(Boolean).at(-1) ?? "";
1209
+ let parsed;
1210
+ try {
1211
+ parsed = JSON.parse(line);
1212
+ }
1213
+ catch {
1214
+ throw new Error("The container's idle stop preparation gave no answer");
1215
+ }
1216
+ return normalizeIdleStopPreparation(parsed, idleTimeoutMs);
1217
+ }
1218
+ /** Releases an idle claim the Hub will not use. Best effort: the claim also expires. */
1219
+ export async function releaseIdleStopInContainer(slug, claimId) {
1220
+ await runDocker(["exec", containerName(slug), "bash", "-c", buildStopClientScript(["--idle-release", claimId])]);
1221
+ }
1222
+ /**
1223
+ * The idle Stop's decision, made before the runner accepts the Stop: the
1224
+ * harness fences only while it still holds the claim and is still idle.
1225
+ * Anything else (the session became active, the claim expired, the
1226
+ * container cannot be reached) refuses the Stop with nothing changed.
1227
+ */
1228
+ export async function fenceIdleStopInContainer(slug, stopOpId, idleClaimId) {
1229
+ try {
1230
+ await runDocker(["exec", containerName(slug), "bash", "-c", buildPrepareDurableStopScript(stopOpId, { fenceOnly: true, idleClaimId })]);
1231
+ return "fenced";
1232
+ }
1233
+ catch {
1234
+ return "superseded";
1235
+ }
1236
+ }
1237
+ /**
1238
+ * Fences the container for this Stop before the runner acknowledges it: the
1239
+ * boot is claimed for the Stop, or a harness already let go is told to spawn
1240
+ * and report nothing more. Never throws; a container that cannot be fenced
1241
+ * is handled by the preparation that follows.
1242
+ */
1243
+ export async function fenceStopInContainer(slug, stopOpId, options = {}) {
1244
+ const script = buildPrepareDurableStopScript(stopOpId, { fenceOnly: true });
1245
+ const deadline = Date.now() + (options.bootSetupWaitMs ?? STOP_BOOT_SETUP_WAIT_MS);
1246
+ for (;;) {
1247
+ try {
1248
+ await runDocker(["exec", containerName(slug), "bash", "-c", script]);
1249
+ return "fenced";
1250
+ }
1251
+ catch (error) {
1252
+ const code = error?.code;
1253
+ if (code === String(STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE) && Date.now() < deadline) {
1254
+ await new Promise((resolve) => setTimeout(resolve, options.retryMs ?? 100));
1255
+ continue;
1256
+ }
1257
+ return "unfenced";
1258
+ }
1259
+ }
1260
+ }
1261
+ /**
1262
+ * Exit codes of /stop-control-client.mjs. Kept in step with that file.
1263
+ * 10: the Stop claimed the container's startup before the harness was let go,
1264
+ * so no agent ever ran there; the entrypoint confirms the save to the Hub
1265
+ * (the Hub already holds the whole workspace) and exits when stopped.
1266
+ * 11: the harness was let go, but stop-control could not be reached or said
1267
+ * the workspace is not hydrated.
1268
+ */
1269
+ export const STOP_CLIENT_STOPPED_BEFORE_RELEASE_EXIT_CODE = 10;
1270
+ export const STOP_CLIENT_STOP_CONTROL_UNAVAILABLE_EXIT_CODE = 11;
1271
+ /** The container's entrypoint has not set up its boot yet; ask again shortly. */
1272
+ export const STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE = 12;
1273
+ /** How long a Stop waits for a just-started container to set up its boot. */
1274
+ export const STOP_BOOT_SETUP_WAIT_MS = 5_000;
1275
+ /**
1276
+ * Exit code of a container whose entrypoint ended a Stop that arrived before
1277
+ * its harness was let go, after the Hub accepted the save confirmation.
1278
+ */
1279
+ export const ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE = 77;
1280
+ /**
1281
+ * Exit code of a container whose entrypoint saved the workspace itself for a
1282
+ * Stop that stop-control never prepared, after the Hub acknowledged the save.
1283
+ */
1284
+ export const ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE = 79;
1285
+ /** Time a container stopped outright gets to confirm the save and report before Docker kills it. */
1286
+ export const OUTRIGHT_STOP_GRACE_SECONDS = 30;
1287
+ /**
1288
+ * Time a prepared container gets to end its harness (bounded at 15 s) and
1289
+ * stop-control (10 s) and report the Stop to the Hub. Docker's implicit 10 s
1290
+ * could kill it before the report, leaving the Hub waiting for its alarm.
1291
+ */
1292
+ export const PREPARED_STOP_GRACE_SECONDS = 30;
1293
+ /** A Scribe's container holds no workspace save; Docker's usual grace. */
1294
+ export const SCRIBE_STOP_GRACE_SECONDS = 10;
1295
+ /**
1296
+ * Longest an inspection waits for the slug's in-flight Stop before answering
1297
+ * (see `waitForInFlightStop`). It covers the seconds between the container
1298
+ * reporting its Stop to the Hub and `docker stop` returning here, and stays
1299
+ * well inside the Hub's 15 s runner-control wait.
1300
+ */
1301
+ export const INSPECTION_STOP_WAIT_MS = 5_000;
1302
+ function dockerExecUnreachable(error) {
1303
+ const code = error?.code;
1304
+ const message = error instanceof Error ? error.message : String(error);
1305
+ return code === "125" || code === "126" || code === "127"
1306
+ || /is not running|no such container|is restarting|is paused|cannot exec|OCI runtime exec failed/i.test(message);
1307
+ }
1308
+ export async function prepareDurableStop(slug, stopOpId, options = {}) {
1309
+ // An idle Stop names its claim so stop-control can reuse the idle save. The
1310
+ // decision was already made at the fence, so a claim stop-control no longer
1311
+ // knows (it restarted) only costs a fresh save, never the Stop.
1312
+ let script = buildPrepareDurableStopScript(stopOpId, { idleClaimId: options.idleClaimId });
1313
+ const deadline = Date.now() + (options.bootSetupWaitMs ?? STOP_BOOT_SETUP_WAIT_MS);
1314
+ for (;;) {
1315
+ try {
1316
+ await runDocker(["exec", containerName(slug), "bash", "-c", script]);
1317
+ return { outcome: "prepared" };
1318
+ }
1319
+ catch (error) {
1320
+ const code = error?.code;
1321
+ const detail = error instanceof Error ? error.message : String(error);
1322
+ if (code === String(STOP_CLIENT_IDLE_CLAIM_SUPERSEDED_EXIT_CODE) && options.idleClaimId) {
1323
+ options = { ...options, idleClaimId: null };
1324
+ script = buildPrepareDurableStopScript(stopOpId);
1325
+ continue;
1326
+ }
1327
+ if (code === String(STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE)) {
1328
+ // The container started a moment ago; its entrypoint sets up the
1329
+ // boot within a few hundred ms, and then the Stop can claim it.
1330
+ if (Date.now() < deadline) {
1331
+ await new Promise((resolve) => setTimeout(resolve, options.retryMs ?? 100));
1332
+ continue;
1333
+ }
1334
+ return { outcome: "stopped-before-release", detail };
1335
+ }
1336
+ if (code === String(STOP_CLIENT_STOPPED_BEFORE_RELEASE_EXIT_CODE))
1337
+ return { outcome: "stopped-before-release", detail };
1338
+ if (code === String(STOP_CLIENT_STOP_CONTROL_UNAVAILABLE_EXIT_CODE) || dockerExecUnreachable(error))
1339
+ return { outcome: "unavailable", detail };
1340
+ // Any other failure, Docker's included: a Stop that was accepted still
1341
+ // ends with the container stopped.
1342
+ return { outcome: "failed", detail };
1343
+ }
1344
+ }
1345
+ }
1346
+ /** Long enough for the environment's own boot to write its base into the mirror. */
1347
+ export const STAGE_AFTER_CREATE_DELAY_MS = 60 * 1000;
1348
+ /**
1349
+ * Long enough for an adopting environment to reach ready (about 3 s after its
1350
+ * container starts) before its replacement's stage run and dependency install
1351
+ * compete with it for CPU and disk. The runner has no ready signal of its own.
1352
+ */
1353
+ export const REFILL_AFTER_ADOPTION_DELAY_MS = 10 * 1000;
1354
+ const RUNNER_VOLUME_RECONCILE_INTERVAL_MS = 24 * 60 * 60 * 1000;
1355
+ export async function startRunnerServer(config) {
1356
+ const { port, image } = config;
1357
+ const identity = Object.freeze({ instanceId: randomUUID(), packageVersion: HOST_PACKAGE_VERSION, runtimeImage: image, runtimePlatform: readRuntimeImagePlatform(image) });
1358
+ const fenceStore = new RunnerCommandFenceStore(config.localStateDir);
1359
+ const localStateDir = config.localStateDir ?? LOCAL_STATE_DIR;
1360
+ const apiToken = config.apiToken || createRunnerApiToken();
1361
+ // One record of every volume the runner manages.
1362
+ const inventory = new RunnerInventory(localStateDir);
1363
+ try {
1364
+ const migrated = inventory.migrate(fenceStore.listWorkspaceVolumeRecords());
1365
+ if (migrated.length)
1366
+ console.log(`[runner] moved ${migrated.map((path) => basename(path)).join(" and ")} into the runner inventory`);
1367
+ }
1368
+ catch (error) {
1369
+ console.error("[runner] could not migrate runner state into the inventory:", error instanceof Error ? error.message : String(error));
1370
+ }
1371
+ const budgetBytes = config.stagedWorkspaceBudgetBytes
1372
+ ?? parseByteSize(process.env[envVar("STAGED_WORKSPACE_BUDGET_BYTES")])
1373
+ ?? STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES;
1374
+ // Staging holds disk ahead of use, so it runs only where there is room:
1375
+ // decided once here from free space on the Docker data volume, and checked
1376
+ // again before each stage run. A budget of 0 (no eviction) is measured
1377
+ // against the default budget.
1378
+ const diskProbe = config.dockerDiskProbe ?? createDockerDiskProbe(image, runDocker);
1379
+ const stagingDecision = decideStagingAtStartup(await diskProbe().catch(() => null), budgetBytes || STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES);
1380
+ const runnerCapabilities = Object.freeze(resolveRunnerCapabilities(image, {
1381
+ stagedWorkspaces: stagingDecision.enabled,
1382
+ terminalLane: Boolean(config.terminalLane),
1383
+ }));
1384
+ // The build-and-test job runs in the runner's own image, given nothing but an app's files.
1385
+ const gadgetJobs = new GadgetJobs({
1386
+ image,
1387
+ localStateDir,
1388
+ runDocker,
1389
+ ...(config.gadgetJob?.runContainer ? { runContainer: config.gadgetJob.runContainer } : {}),
1390
+ ...(config.gadgetJob?.timeoutMs ? { timeoutMs: config.gadgetJob.timeoutMs } : {}),
1391
+ log: (message) => console.log(message),
1392
+ });
1393
+ const terminalLaneTickets = new TerminalLaneTickets();
1394
+ const terminalLaneRelays = new TerminalLaneRelays();
1395
+ /** Stop and Delete end every lane of the environment and void its unused tickets. */
1396
+ const closeTerminalLanes = (slug) => {
1397
+ terminalLaneTickets.revoke(slug);
1398
+ terminalLaneRelays.closeAll(slug);
1399
+ };
1400
+ const stagedWorkspaces = stagingDecision.enabled
1401
+ ? new StagedWorkspaces({
1402
+ inventory,
1403
+ image,
1404
+ runDocker,
1405
+ canonicalRepositoryUrl: canonicalGitHubRepositoryUrl,
1406
+ mirrorVolumeName,
1407
+ dependencies: { cacheMounts: PACKAGE_CACHE_MOUNTS, maxBytes: budgetBytes },
1408
+ diskRefusal: async () => stageRunDiskRefusal(await diskProbe().catch(() => null)),
1409
+ })
1410
+ : null;
1411
+ /**
1412
+ * Pending delayed stages by `repo@base`, with the environments waiting on
1413
+ * each. `null` stands for one no Stop or Delete may cancel.
1414
+ */
1415
+ const stageTimers = new Map();
1416
+ const refillTimers = new Set();
1417
+ // Development-only options, honoured only while the Hub says it is a
1418
+ // development build (`development-runner.ts`).
1419
+ const development = { ...readDevelopmentRunnerOptions(), ...config.development };
1420
+ const developmentHub = new DevelopmentHubGate(config.hubBuildChannel ?? (() => fetchHubBuildChannel(HUB_URL, hubControlHeaders)));
1421
+ const developmentKnobs = development.stageDelayMs !== null || development.refillDelayMs !== null;
1422
+ const stagingHold = new StagingHold();
1423
+ if (developmentKnobs || development.harnessDir) {
1424
+ void developmentHub.isDevelopment().then((isDevelopment) => {
1425
+ const names = [
1426
+ development.harnessDir ? `${envVar("DEV_HARNESS_DIR")}=${development.harnessDir}` : null,
1427
+ development.stageDelayMs !== null ? `${envVar("STAGE_DELAY_MS")}=${development.stageDelayMs}` : null,
1428
+ development.refillDelayMs !== null ? `${envVar("REFILL_DELAY_MS")}=${development.refillDelayMs}` : null,
1429
+ ].filter(Boolean).join(", ");
1430
+ console.log(isDevelopment
1431
+ ? `[runner] development options on (the Hub is a development build): ${names}`
1432
+ : `[runner] development options ignored, the Hub is not a known development build: ${names}`);
1433
+ });
1434
+ }
1435
+ /** A development delay when one is set and the Hub was last seen as a development build. */
1436
+ const developmentDelay = (value, fallback) => value !== null && developmentHub.knownDevelopment() ? value : fallback;
1437
+ async function resolveHarnessOverlay() {
1438
+ if (!development.harnessDir)
1439
+ return null;
1440
+ if (!(await developmentHub.isDevelopment()))
1441
+ throw new Error(HARNESS_OVERLAY_REFUSED);
1442
+ return validateHarnessOverlayDir(development.harnessDir);
1443
+ }
1444
+ const preparations = new Map();
1445
+ const gitMirrorTimers = [];
1446
+ /** Bookkeeping never fails a start; a missed update is repaired by the next reconciliation. */
1447
+ const updateInventory = (change) => {
1448
+ try {
1449
+ change(inventory);
1450
+ }
1451
+ catch (error) {
1452
+ console.error("[runner] could not update the runner inventory:", error instanceof Error ? error.message : String(error));
1453
+ }
1454
+ };
1455
+ const commandTails = new Map();
1456
+ const commandExecutions = new Map();
1457
+ const stopAcknowledgements = new Map();
1458
+ /** The Stop each slug is still executing, settled either way. */
1459
+ const inFlightStops = new Map();
1460
+ /** The idle claim an accepted idle Stop decided on, for its prepare (save reuse). */
1461
+ const idleStopClaims = new Map();
1462
+ function commandKey(record) {
1463
+ return `${record.slug}:${record.commandGeneration}:${record.operationId}:${record.desiredState}`;
1464
+ }
1465
+ function commandResponse(record) {
1466
+ return {
1467
+ commandGeneration: record.commandGeneration,
1468
+ operationId: record.operationId,
1469
+ desiredState: record.desiredState,
1470
+ };
1471
+ }
1472
+ function scheduleCommand(record, effect) {
1473
+ const key = commandKey(record);
1474
+ const existing = commandExecutions.get(key);
1475
+ if (existing)
1476
+ return existing;
1477
+ const previous = commandTails.get(record.slug) ?? Promise.resolve();
1478
+ const execution = previous.catch(() => { }).then(effect);
1479
+ const tail = execution.then(() => undefined, () => undefined);
1480
+ commandTails.set(record.slug, tail);
1481
+ commandExecutions.set(key, execution);
1482
+ void execution.then(() => {
1483
+ if (commandExecutions.get(key) === execution)
1484
+ commandExecutions.delete(key);
1485
+ }, () => {
1486
+ if (commandExecutions.get(key) === execution)
1487
+ commandExecutions.delete(key);
1488
+ });
1489
+ void tail.then(() => {
1490
+ if (commandTails.get(record.slug) === tail)
1491
+ commandTails.delete(record.slug);
1492
+ });
1493
+ return execution;
1494
+ }
1495
+ function acceptCommand(slug, payload, desiredState) {
1496
+ return fenceStore.accept(slug, commandEnvelopeFromPayload(payload), desiredState);
1497
+ }
1498
+ /**
1499
+ * A Start or Create for a slug whose Stop is still executing waits for that
1500
+ * Stop to finish before it is accepted. The Hub calls an environment
1501
+ * stopped as soon as its container reports the Stop, while `docker stop`
1502
+ * can still take seconds to return here; accepting the Start then would
1503
+ * cut the Stop off at its next fence check, before it records the proof
1504
+ * that lets this Start reuse the container. Waiting changes nothing else:
1505
+ * the per-slug queue would run the Start after the Stop anyway, and a
1506
+ * Start older than the Stop is still refused by the fence once accepted.
1507
+ * An inspection waits the same way, but only up to `boundMs`, and then
1508
+ * answers with whatever it sees.
1509
+ */
1510
+ async function waitForInFlightStop(slug, action, boundMs) {
1511
+ if (!inFlightStops.has(slug))
1512
+ return;
1513
+ const startedAt = performance.now();
1514
+ let waited = false;
1515
+ let timer;
1516
+ const bound = boundMs === undefined
1517
+ ? null
1518
+ : new Promise((resolvePromise) => { timer = setTimeout(() => resolvePromise("timed-out"), boundMs); });
1519
+ try {
1520
+ for (let stop = inFlightStops.get(slug); stop; stop = inFlightStops.get(slug)) {
1521
+ waited = true;
1522
+ if (await (bound ? Promise.race([stop, bound]) : stop) === "timed-out") {
1523
+ console.log(`[runner] ${action} ${slug} stopped waiting for its in-flight Stop after ${Math.round(performance.now() - startedAt)} ms`);
1524
+ return;
1525
+ }
1526
+ }
1527
+ }
1528
+ finally {
1529
+ if (timer)
1530
+ clearTimeout(timer);
1531
+ }
1532
+ if (waited) {
1533
+ console.log(`[runner] ${action} ${slug} waited ${Math.round(performance.now() - startedAt)} ms for its in-flight Stop to finish`);
1534
+ }
1535
+ }
1536
+ /**
1537
+ * The runner's backstop for the Hub's own check: refuses a Create or Start
1538
+ * past the environment limit, before the command is accepted. Never refuses
1539
+ * an isolated run, a retry of the command already accepted for the slug, a
1540
+ * container already running, or when the limit or the count is unknown.
1541
+ */
1542
+ async function refuseWhenFull(slug, payload) {
1543
+ const limit = config.environmentLimit?.() ?? null;
1544
+ if (limit === null)
1545
+ return;
1546
+ const envVars = payload.envVars && typeof payload.envVars === "object"
1547
+ ? payload.envVars
1548
+ : {};
1549
+ if (isIsolatedRun(envVars))
1550
+ return;
1551
+ const envelope = commandEnvelopeFromPayload(payload);
1552
+ const accepted = fenceStore.read(slug);
1553
+ if (accepted
1554
+ && envelope.commandGeneration !== undefined
1555
+ && accepted.commandGeneration === envelope.commandGeneration
1556
+ && accepted.operationId === envelope.operationId)
1557
+ return;
1558
+ const count = await (config.countRunningEnvironments ?? (() => countRunningEnvironments(runCapacityDocker)))();
1559
+ if (!isMachineFull(count, limit))
1560
+ return;
1561
+ const info = await inspectContainer(slug).catch(() => null);
1562
+ if (info && getStatus(info) === "running")
1563
+ return;
1564
+ throw new MachineFullError(count, limit);
1565
+ }
1566
+ async function executeFencedStart(record, payload, action, timer = createStepTimer()) {
1567
+ timer.mark("queue");
1568
+ const guard = () => fenceStore.assertCurrent(record);
1569
+ const fencedPayload = {
1570
+ ...payload,
1571
+ commandGeneration: record.commandGeneration,
1572
+ operationId: record.operationId,
1573
+ desiredState: record.desiredState,
1574
+ };
1575
+ const context = {
1576
+ guard,
1577
+ fenceDirectory: fenceStore.directoryFor(record.slug),
1578
+ readWorkspaceSyncedStop: () => fenceStore.readWorkspaceSyncedStop(record.slug),
1579
+ readLaunched: () => fenceStore.readLaunched(record.slug),
1580
+ recordLaunch: (containerId, envFileContents) => {
1581
+ fenceStore.writeLaunchFile(record, envFileContents);
1582
+ fenceStore.recordLaunched(record, containerId);
1583
+ },
1584
+ resolveGitMirror: (mirrorPayload) => resolveGitMirrorMount(mirrorPayload, localStateDir, mirrorPresence),
1585
+ terminalLane: config.terminalLane ?? null,
1586
+ resolveWorkspaceVolume: (volumePayload, state) => resolveWorkspaceVolume(record, volumePayload, state),
1587
+ mark: timer.mark,
1588
+ resolveHarnessOverlay,
1589
+ };
1590
+ // Keeps the development answer fresh for the staging delays.
1591
+ if (developmentKnobs)
1592
+ void developmentHub.isDevelopment();
1593
+ const result = action === "create"
1594
+ ? await createContainer(fencedPayload, image, context)
1595
+ : await ensureStarted(record.slug, fencedPayload, image, context);
1596
+ guard();
1597
+ fenceStore.markApplied(record);
1598
+ timer.mark("mark-applied");
1599
+ console.log(`[runner] ${action} ${record.slug} took ${timer.line()}`);
1600
+ if (result.workspaceVolumeKind === "staged")
1601
+ refillAfterAdoption(fencedPayload);
1602
+ else if (result.workspaceVolumeKind)
1603
+ scheduleStagingAfterCreate(fencedPayload, record);
1604
+ return result;
1605
+ }
1606
+ /**
1607
+ * The slug's recorded volume; otherwise, with staging on, a staged volume
1608
+ * adopted now or the default name, recorded before `docker create`. With
1609
+ * staging off nothing is recorded, so switching it off needs no migration,
1610
+ * while a slug that adopted a volume keeps it.
1611
+ */
1612
+ async function resolveWorkspaceVolume(record, payload, state) {
1613
+ const recorded = fenceStore.readWorkspaceVolume(record.slug);
1614
+ if (recorded) {
1615
+ updateInventory((entries) => entries.recordWorkspace(record.slug, recorded));
1616
+ return { volume: recorded, kind: "existing" };
1617
+ }
1618
+ const fallback = workspaceVolumeName(record.slug);
1619
+ let kind = state.containerExists ? "existing" : "fresh";
1620
+ if (!stagedWorkspaces)
1621
+ return { volume: fallback, kind };
1622
+ stagedWorkspaces.noteDependencyPreference(payload.repoUrl, payload.envVars);
1623
+ const held = stagingHold.active();
1624
+ // Only a slug that has no workspace yet adopts. A legacy slug's default
1625
+ // volume may hold work that never reached the Hub.
1626
+ if (kind === "fresh" && payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined) {
1627
+ if (await volumeExists(fallback)) {
1628
+ kind = "existing";
1629
+ }
1630
+ else if (held) {
1631
+ console.log(`[runner] ${record.slug} adopts no staged workspace: staging is held for the startup bench`);
1632
+ }
1633
+ else {
1634
+ const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
1635
+ const adopted = await stagedWorkspaces.adopt(payload.repoUrl, base, record.slug, (volume) => fenceStore.writeWorkspaceVolume(record.slug, volume), () => fenceStore.assertCurrent(record));
1636
+ if (adopted) {
1637
+ console.log(`[runner] ${record.slug} adopted staged workspace ${adopted.volume} at ${adopted.commit.slice(0, 12)}`
1638
+ + (adopted.exact ? "" : `; the bootstrap updates it to ${base?.slice(0, 12)}`));
1639
+ return {
1640
+ volume: adopted.volume,
1641
+ kind: "staged",
1642
+ ...(adopted.deps ? { stagedDependencies: adopted.deps } : {}),
1643
+ };
1644
+ }
1645
+ }
1646
+ }
1647
+ fenceStore.assertCurrent(record);
1648
+ const volume = fenceStore.writeWorkspaceVolume(record.slug, fallback);
1649
+ updateInventory((entries) => entries.recordWorkspace(record.slug, volume));
1650
+ return { volume, kind: volume === fallback ? kind : "existing" };
1651
+ }
1652
+ function stageAndLog(repoUrl, commit, trigger, options = {}) {
1653
+ if (!stagedWorkspaces)
1654
+ return Promise.resolve("skipped");
1655
+ if (stagingHold.active()) {
1656
+ console.log(`[runner] not staging a workspace for ${repoUrl} (${trigger}): staging is held for the startup bench`);
1657
+ return Promise.resolve("skipped");
1658
+ }
1659
+ return stagedWorkspaces.stage(repoUrl, commit, { replace: options.replace }).then(({ result, done }) => {
1660
+ void done.then((outcome) => {
1661
+ if (outcome.kind === "skipped" && result === "skipped") {
1662
+ console.log(`[runner] not staging a workspace for ${repoUrl} (${trigger}): ${outcome.reason}`);
1663
+ }
1664
+ if (outcome.kind === "commit-missing")
1665
+ options.onCommitMissing?.();
1666
+ });
1667
+ return result;
1668
+ }, (error) => {
1669
+ console.error(`[runner] staging a workspace for ${repoUrl} failed (${trigger}):`, error instanceof Error ? error.message : String(error));
1670
+ return "skipped";
1671
+ });
1672
+ }
1673
+ /**
1674
+ * Replaces a staged volume shortly after a create has adopted it, once the
1675
+ * adopting environment has had time to reach ready, so the stage run does
1676
+ * not slow that start. The mirror held the adopted commit, so when the base
1677
+ * is that commit the stage run can go ahead whatever happens to the
1678
+ * adopting environment: a Stop or Delete does not cancel it. When the
1679
+ * volume was at an older commit the mirror may not hold the base yet; that
1680
+ * run ends as commit-missing and the delayed path takes over.
1681
+ */
1682
+ function refillAfterAdoption(payload) {
1683
+ if (!stagedWorkspaces)
1684
+ return;
1685
+ const repository = canonicalGitHubRepositoryUrl(payload.repoUrl);
1686
+ const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
1687
+ if (!repository || !base)
1688
+ return;
1689
+ const timer = setTimeout(() => {
1690
+ refillTimers.delete(timer);
1691
+ void stageAndLog(repository, base, "after adoption", {
1692
+ onCommitMissing: () => scheduleStagingAfterCreate(payload),
1693
+ });
1694
+ }, config.refillAfterAdoptionDelayMs ?? developmentDelay(development.refillDelayMs, REFILL_AFTER_ADOPTION_DELAY_MS));
1695
+ timer.unref();
1696
+ refillTimers.add(timer);
1697
+ }
1698
+ /**
1699
+ * Replenishes the repository's staged volume a minute after a create that
1700
+ * adopted none, once the environment's own boot has written its base into
1701
+ * the mirror. Creates of the same repository and base within that minute
1702
+ * share one timer, which stages when any of them is still current: with
1703
+ * `record`, a Stop or Delete of that one environment only withdraws it, and
1704
+ * without, nothing cancels it. When the run finds the mirror without the
1705
+ * base (the first environment left before its boot fetched it), environments
1706
+ * still current get one more full delay.
1707
+ */
1708
+ function scheduleStagingAfterCreate(payload, record, retry = false) {
1709
+ if (!stagedWorkspaces)
1710
+ return;
1711
+ const repository = canonicalGitHubRepositoryUrl(payload.repoUrl);
1712
+ const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
1713
+ if (!repository || !base)
1714
+ return;
1715
+ const key = `${repository}@${base}`;
1716
+ const pending = stageTimers.get(key);
1717
+ if (pending) {
1718
+ pending.records.push(record ?? null);
1719
+ return;
1720
+ }
1721
+ const records = [record ?? null];
1722
+ const timer = setTimeout(() => {
1723
+ stageTimers.delete(key);
1724
+ const live = records.filter((waiting) => {
1725
+ if (!waiting)
1726
+ return true;
1727
+ try {
1728
+ fenceStore.assertCurrent(waiting);
1729
+ return true;
1730
+ }
1731
+ catch {
1732
+ return false;
1733
+ }
1734
+ });
1735
+ if (!live.length) {
1736
+ console.log(`[runner] not staging a workspace for ${repository} (after create): every environment that asked was stopped or deleted`);
1737
+ return;
1738
+ }
1739
+ void stageAndLog(repository, base, "after create", {
1740
+ onCommitMissing: retry ? undefined : () => {
1741
+ for (const waiting of live)
1742
+ scheduleStagingAfterCreate(payload, waiting ?? undefined, true);
1743
+ },
1744
+ });
1745
+ }, config.stageAfterCreateDelayMs ?? developmentDelay(development.stageDelayMs, STAGE_AFTER_CREATE_DELAY_MS));
1746
+ timer.unref();
1747
+ stageTimers.set(key, { timer, records });
1748
+ }
1749
+ /**
1750
+ * Refreshes the repository's mirror in a short-lived root container that
1751
+ * holds only the Hub's scoped, read-only bridge credential, then starts
1752
+ * staging a workspace at that commit, in place of one staged at another
1753
+ * head. Concurrent requests for the same
1754
+ * mirror and commit share one run.
1755
+ */
1756
+ function prepareProject(request) {
1757
+ const repository = canonicalGitHubRepositoryUrl(request.repoUrl);
1758
+ const mirrorVolume = repository ? mirrorVolumeName(repository) : null;
1759
+ if (!repository || !mirrorVolume) {
1760
+ return Promise.resolve({ ok: true, mirror: "skipped", staged: "skipped" });
1761
+ }
1762
+ const env = buildMirrorRefreshEnv(request, repository);
1763
+ const mirrorOnly = request.stage === false;
1764
+ const key = `${mirrorVolume}@${request.headSha}${mirrorOnly ? ":mirror" : ""}`;
1765
+ const existing = preparations.get(key);
1766
+ if (existing)
1767
+ return existing;
1768
+ const preparation = (async () => {
1769
+ try {
1770
+ recordGitMirrorUse(mirrorVolume, repository, localStateDir);
1771
+ }
1772
+ catch (error) {
1773
+ console.error(`[runner] could not record Git mirror use for ${mirrorVolume}:`, error instanceof Error ? error.message : String(error));
1774
+ }
1775
+ const envFile = createDockerEnvFile(env);
1776
+ const refreshName = dockerName(`mirror-refresh-${randomUUID()}`);
1777
+ let timedOut = false;
1778
+ const timer = setTimeout(() => {
1779
+ timedOut = true;
1780
+ void runDocker(["rm", "-f", refreshName]).catch(() => { });
1781
+ }, MIRROR_REFRESH_CONTAINER_TIMEOUT_MS);
1782
+ timer.unref();
1783
+ let output;
1784
+ try {
1785
+ output = await runDocker(buildMirrorRefreshRunArgs({
1786
+ image,
1787
+ containerName: refreshName,
1788
+ mirrorVolume,
1789
+ envFilePath: envFile.filePath,
1790
+ }));
1791
+ }
1792
+ catch (error) {
1793
+ throw redactError(timedOut ? new Error("The mirror refresh timed out") : error, Object.fromEntries(PREPARE_PROJECT_SECRET_NAMES.flatMap((name) => env[name] ? [[name, env[name]]] : [])));
1794
+ }
1795
+ finally {
1796
+ clearTimeout(timer);
1797
+ envFile.cleanup();
1798
+ }
1799
+ const mirror = parseMirrorRefreshOutcome(output);
1800
+ if (!mirror)
1801
+ throw new Error("The mirror refresh reported no outcome");
1802
+ // A plan's basis commit only needs to be in the mirror; the staged
1803
+ // workspace stays at the default-branch head.
1804
+ if (mirrorOnly) {
1805
+ console.log(`[runner] mirrored ${repository} at ${request.headSha.slice(0, 12)}: mirror=${mirror}`);
1806
+ return { ok: true, mirror, staged: "skipped" };
1807
+ }
1808
+ // The mirror holds the head now, so a volume staged at an older head is replaced.
1809
+ const staged = await stageAndLog(repository, request.headSha, "project prepare", { replace: true });
1810
+ console.log(`[runner] prepared ${repository} at ${request.headSha.slice(0, 12)}: mirror=${mirror} staged=${staged}`);
1811
+ return { ok: true, mirror, staged };
1812
+ })();
1813
+ preparations.set(key, preparation);
1814
+ void preparation.then(() => { if (preparations.get(key) === preparation)
1815
+ preparations.delete(key); }, () => { if (preparations.get(key) === preparation)
1816
+ preparations.delete(key); });
1817
+ return preparation;
1818
+ }
1819
+ // Spares isolated creates a `docker volume inspect` (see VolumePresenceCache).
1820
+ const mirrorPresence = new VolumePresenceCache({
1821
+ inspect: gitMirrorVolumeExists,
1822
+ known: (volume) => readRunnerInventory(localStateDir)[volume]?.kind === "mirror",
1823
+ });
1824
+ /**
1825
+ * The one cleanup pass over the volumes the runner manages (see
1826
+ * `reconcileRunnerVolumes`), at startup and then daily.
1827
+ */
1828
+ function reconcileVolumes() {
1829
+ void reconcileRunnerVolumes({
1830
+ inventory,
1831
+ runDocker,
1832
+ workspaceRecords: () => fenceStore.listWorkspaceVolumeRecords(),
1833
+ defaultWorkspaceVolume: workspaceVolumeName,
1834
+ mirrors: true,
1835
+ staged: stagedWorkspaces
1836
+ ? { isBuilding: (volume) => stagedWorkspaces.isBuilding(volume), budgetBytes }
1837
+ : null,
1838
+ }).then(({ removed }) => {
1839
+ mirrorPresence.forget(removed);
1840
+ if (removed.length)
1841
+ console.log(`[runner] removed unused runner volumes: ${removed.join(", ")}`);
1842
+ }, (error) => {
1843
+ console.error("[runner] runner volume cleanup failed:", error instanceof Error ? error.message : String(error));
1844
+ });
1845
+ }
1846
+ async function executeFencedStop(record, onInspected) {
1847
+ closeTerminalLanes(record.slug);
1848
+ fenceStore.assertCurrent(record);
1849
+ const info = await inspectContainer(record.slug, true);
1850
+ fenceStore.assertCurrent(record);
1851
+ const launched = fenceStore.readLaunched(record.slug);
1852
+ if (neverStarted(info)) {
1853
+ // A create superseded between `docker create` and `docker start` (a
1854
+ // Stop sent about a second after the create returned), or a runner that
1855
+ // died there: Docker made the container but no process in it ever ran,
1856
+ // so it never touched the workspace and there is nothing to save. The
1857
+ // container is left as it is: the next Start's create removes a
1858
+ // `created` container no current command claims, and a Delete removes
1859
+ // it with the rest.
1860
+ fenceStore.removeLaunchFiles(record.slug);
1861
+ fenceStore.markApplied(record);
1862
+ onInspected({ callbackExpected: false, startRejectedBeforeWorkspace: true });
1863
+ return;
1864
+ }
1865
+ if (!info || getStatus(info) !== "running") {
1866
+ if (info && !["stopped", "exited", "dead"].includes(getStatus(info))) {
1867
+ throw new Error("The container is not confirmed stopped. Its retained data was left untouched.");
1868
+ }
1869
+ // Either the entrypoint refused the superseded start before touching
1870
+ // the workspace, or the Stop superseded the slug's first start before
1871
+ // Docker created a container (a Stop sent right after the create
1872
+ // returns): no container was ever launched here, so nothing can hold
1873
+ // unsaved work.
1874
+ const startRejectedBeforeWorkspace = info
1875
+ ? info.State?.ExitCode === ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE
1876
+ && launchedGeneration(info, launched) === record.commandGeneration - 1
1877
+ : launched === null;
1878
+ fenceStore.removeLaunchFiles(record.slug);
1879
+ fenceStore.markApplied(record);
1880
+ onInspected({
1881
+ callbackExpected: false,
1882
+ ...(startRejectedBeforeWorkspace ? { startRejectedBeforeWorkspace: true } : {}),
1883
+ });
1884
+ return;
1885
+ }
1886
+ const bootstrapModes = (info.Config?.Env ?? []).filter(value => value.startsWith("TILLER_BOOTSTRAP_MODE="));
1887
+ if (bootstrapModes.length === 1 && bootstrapModes[0] === "TILLER_BOOTSTRAP_MODE=plan-writer") {
1888
+ if (launchedGeneration(info, launched) !== record.commandGeneration - 1)
1889
+ throw new Error("Scribe retention stop does not match its launched container generation");
1890
+ await runDocker(["stop", "-t", String(SCRIBE_STOP_GRACE_SECONDS), containerName(record.slug)]);
1891
+ fenceStore.assertCurrent(record);
1892
+ const stopped = await inspectContainer(record.slug, true);
1893
+ fenceStore.assertCurrent(record);
1894
+ if (!stopped || !["stopped", "exited", "dead"].includes(getStatus(stopped)))
1895
+ throw new Error("Scribe stopped-container retention was not confirmed");
1896
+ fenceStore.removeLaunchFiles(record.slug);
1897
+ fenceStore.markApplied(record);
1898
+ onInspected({ callbackExpected: false });
1899
+ return;
1900
+ }
1901
+ // Fence first, acknowledge after: once the Hub hears the Stop was
1902
+ // accepted, no agent can be spawned and nothing reports ready.
1903
+ const fence = await fenceStopInContainer(record.slug, record.operationId);
1904
+ fenceStore.assertCurrent(record);
1905
+ if (fence === "unfenced")
1906
+ console.error(`[runner] could not fence ${record.slug} before acknowledging the Stop; preparing it anyway`);
1907
+ onInspected({ callbackExpected: true });
1908
+ const { outcome: preparation, detail: preparationDetail } = await prepareDurableStop(record.slug, record.operationId, {
1909
+ idleClaimId: idleStopClaims.get(commandKey(record)) ?? null,
1910
+ });
1911
+ fenceStore.assertCurrent(record);
1912
+ const runningGeneration = launchedGeneration(info, launched);
1913
+ if (preparation !== "prepared") {
1914
+ // Nothing saved the workspace through stop-control: the Stop came
1915
+ // before the harness was let go, or stop-control cannot be reached.
1916
+ // Either way an accepted Stop must not leave an agent running, so the
1917
+ // container is stopped outright. Its entrypoint never uploads a
1918
+ // partial workspace; before release it confirms to the Hub that the
1919
+ // Hub's copy is the whole workspace and exits with a dedicated code.
1920
+ console.error(`[runner] stop preparation for ${record.slug} was ${preparation}${preparationDetail ? ` (${preparationDetail})` : ""}; stopping the container outright`);
1921
+ await runDocker(["stop", "-t", String(OUTRIGHT_STOP_GRACE_SECONDS), containerName(record.slug)]);
1922
+ fenceStore.assertCurrent(record);
1923
+ const stopped = await inspectContainer(record.slug, true);
1924
+ fenceStore.assertCurrent(record);
1925
+ if (stopped && !["stopped", "exited", "dead"].includes(getStatus(stopped))) {
1926
+ throw new Error("Docker did not confirm the container stopped.");
1927
+ }
1928
+ // Only a container that ended the Stop before its harness ran, and had
1929
+ // the Hub accept that, proves nothing is left to save; the next Start
1930
+ // may then replace it. Anything else stays for the user to decide.
1931
+ // A container that saved for itself after the preparation failed
1932
+ // (exit 79) proves the same. The exit code is the entrypoint's, which
1933
+ // the workload cannot set.
1934
+ const sameContainer = !info.Id || stopped?.Id === info.Id;
1935
+ const confirmedBeforeRelease = sameContainer && ((preparation === "stopped-before-release"
1936
+ && stopped?.State?.ExitCode === ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE)
1937
+ || stopped?.State?.ExitCode === ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE);
1938
+ if (confirmedBeforeRelease && (runningGeneration !== null || info.Id)) {
1939
+ fenceStore.recordWorkspaceSyncedStop(record, runningGeneration, info.Id);
1940
+ }
1941
+ fenceStore.removeLaunchFiles(record.slug);
1942
+ fenceStore.markApplied(record);
1943
+ if (!confirmedBeforeRelease) {
1944
+ throw new Error(`The container for ${record.slug} was stopped without a confirmed workspace save (${preparation}${preparationDetail ? `: ${preparationDetail}` : ""}).`);
1945
+ }
1946
+ return;
1947
+ }
1948
+ // A durable stop of this container is proof for this container even when
1949
+ // the launch record no longer attributes it, so the next Start can still
1950
+ // replace it instead of wedging.
1951
+ if (runningGeneration !== null || info.Id) {
1952
+ fenceStore.recordWorkspaceSyncedStop(record, runningGeneration, info.Id);
1953
+ }
1954
+ await runDocker(["stop", "-t", String(PREPARED_STOP_GRACE_SECONDS), containerName(record.slug)]);
1955
+ fenceStore.assertCurrent(record);
1956
+ const stopped = await inspectContainer(record.slug, true);
1957
+ fenceStore.assertCurrent(record);
1958
+ if (stopped && !["stopped", "exited", "dead"].includes(getStatus(stopped))) {
1959
+ throw new Error("Docker did not confirm the container stopped.");
1960
+ }
1961
+ fenceStore.removeLaunchFiles(record.slug);
1962
+ fenceStore.markApplied(record);
1963
+ }
1964
+ function scheduleFencedStop(record) {
1965
+ const key = commandKey(record);
1966
+ const existing = stopAcknowledgements.get(key);
1967
+ if (existing)
1968
+ return existing;
1969
+ let acknowledgementSettled = false;
1970
+ let resolveAcknowledgement;
1971
+ let rejectAcknowledgement;
1972
+ const acknowledgement = new Promise((resolvePromise, rejectPromise) => {
1973
+ resolveAcknowledgement = resolvePromise;
1974
+ rejectAcknowledgement = rejectPromise;
1975
+ });
1976
+ // Recovery schedules Stop without an HTTP waiter; keep a rejected early
1977
+ // inspection from becoming an unhandled promise while preserving it for
1978
+ // request callers that do await the acknowledgement.
1979
+ void acknowledgement.catch(() => { });
1980
+ const completion = scheduleCommand(record, () => executeFencedStop(record, (result) => {
1981
+ if (acknowledgementSettled)
1982
+ return;
1983
+ acknowledgementSettled = true;
1984
+ resolveAcknowledgement(result);
1985
+ }));
1986
+ const scheduled = { acknowledgement, completion };
1987
+ stopAcknowledgements.set(key, scheduled);
1988
+ const settledStop = completion.then(() => undefined, () => undefined);
1989
+ inFlightStops.set(record.slug, settledStop);
1990
+ void settledStop.then(() => {
1991
+ if (inFlightStops.get(record.slug) === settledStop)
1992
+ inFlightStops.delete(record.slug);
1993
+ });
1994
+ void completion.then(() => {
1995
+ if (!acknowledgementSettled) {
1996
+ acknowledgementSettled = true;
1997
+ resolveAcknowledgement({ callbackExpected: false });
1998
+ }
1999
+ if (stopAcknowledgements.get(key) === scheduled)
2000
+ stopAcknowledgements.delete(key);
2001
+ }, (error) => {
2002
+ if (!acknowledgementSettled) {
2003
+ acknowledgementSettled = true;
2004
+ rejectAcknowledgement(error);
2005
+ }
2006
+ if (stopAcknowledgements.get(key) === scheduled)
2007
+ stopAcknowledgements.delete(key);
2008
+ });
2009
+ return scheduled;
2010
+ }
2011
+ /**
2012
+ * An idle Stop is decided before it is accepted: the harness must still
2013
+ * hold the claim (see fenceIdleStopInContainer). A refusal accepts nothing,
2014
+ * so the Hub can put the session back to running. A retry of a Stop this
2015
+ * runner already accepted is not asked again, and a container that is not
2016
+ * running has nothing to keep: its Stop goes ahead as any other.
2017
+ */
2018
+ async function admitIdleStop(slug, payload, idleClaimId) {
2019
+ const envelope = commandEnvelopeFromPayload(payload);
2020
+ const operationId = typeof envelope.operationId === "string" ? envelope.operationId.trim() : "";
2021
+ const current = fenceStore.read(slug);
2022
+ if (current && current.commandGeneration === envelope.commandGeneration && current.operationId === operationId)
2023
+ return true;
2024
+ if (!operationId)
2025
+ return false;
2026
+ const info = await inspectContainer(slug, true);
2027
+ if (!info || getStatus(info) !== "running")
2028
+ return true;
2029
+ const decided = await fenceIdleStopInContainer(slug, operationId, idleClaimId);
2030
+ console.log(`[runner] idle stop for ${slug} ${decided === "fenced" ? "decided: the harness still held its claim" : "refused: the session is active or its claim is gone"}`);
2031
+ return decided === "fenced";
2032
+ }
2033
+ async function executeFencedDestroy(record) {
2034
+ fenceStore.assertCurrent(record);
2035
+ closeTerminalLanes(record.slug);
2036
+ const removed = await forceRemoveContainer(record.slug);
2037
+ fenceStore.assertCurrent(record);
2038
+ if (await inspectContainer(record.slug, true)) {
2039
+ throw new Error("Docker did not confirm the container was removed.");
2040
+ }
2041
+ let recordedVolume = null;
2042
+ try {
2043
+ recordedVolume = fenceStore.readWorkspaceVolume(record.slug);
2044
+ }
2045
+ catch (error) {
2046
+ // The default volume still goes; an unreadable record's staged volume
2047
+ // is left for the staged-workspace cleanup.
2048
+ console.error(`[runner] ${record.slug}:`, error instanceof Error ? error.message : String(error));
2049
+ }
2050
+ const removedWorkspaceVolume = await removeWorkspaceVolumes(record.slug, recordedVolume, image);
2051
+ fenceStore.assertCurrent(record);
2052
+ fenceStore.removeWorkspaceVolume(record.slug);
2053
+ updateInventory((entries) => entries.forgetWorkspace(record.slug));
2054
+ const removedOpencodeState = await removeLocalOpencodeStateForRunner(record.slug, image);
2055
+ fenceStore.assertCurrent(record);
2056
+ fenceStore.removeLaunchFiles(record.slug);
2057
+ fenceStore.removeLaunched(record.slug);
2058
+ fenceStore.markApplied(record);
2059
+ return { removed: removed || removedWorkspaceVolume || removedOpencodeState };
2060
+ }
2061
+ function recoverAcceptedTerminalCommands() {
2062
+ for (const record of fenceStore.listAcceptedTerminalCommands()) {
2063
+ const execution = record.desiredState === "stopped"
2064
+ ? scheduleFencedStop(record).completion
2065
+ : scheduleCommand(record, () => executeFencedDestroy(record));
2066
+ void execution.catch((error) => {
2067
+ console.error(`[runner] Failed to recover ${record.desiredState} command for ${record.slug}:`, error instanceof Error ? error.message : String(error));
2068
+ });
2069
+ }
2070
+ }
2071
+ /**
2072
+ * Development only: `POST /dev/staging-hold` with `{ "hold": true, "ms": N }`
2073
+ * holds staging for N ms (default 10 min, at most an hour), `{ "hold": false }`
2074
+ * ends it. The startup bench holds it so `mirror-hit` starts never adopt.
2075
+ */
2076
+ async function handleStagingHold(req, res) {
2077
+ if (req.method !== "POST") {
2078
+ methodNotAllowed(res);
2079
+ return;
2080
+ }
2081
+ try {
2082
+ const body = await readJson(req);
2083
+ if (!(await developmentHub.isDevelopment())) {
2084
+ json(res, 403, { ok: false, error: "The staging hold is for development Hubs only." });
2085
+ return;
2086
+ }
2087
+ if (body.hold === false) {
2088
+ stagingHold.release();
2089
+ console.log("[runner] staging hold ended");
2090
+ json(res, 200, { ok: true, held: false, staging: stagedWorkspaces ? "on" : "off" });
2091
+ return;
2092
+ }
2093
+ if (body.hold !== true || (body.ms !== undefined && (!Number.isSafeInteger(body.ms) || body.ms < 0))) {
2094
+ json(res, 400, { ok: false, error: "Expected { \"hold\": true, \"ms\"?: <non-negative integer> } or { \"hold\": false }." });
2095
+ return;
2096
+ }
2097
+ const until = stagingHold.hold(body.ms ?? STAGING_HOLD_DEFAULT_MS);
2098
+ console.log(`[runner] staging held for the startup bench until ${new Date(until).toISOString()}`);
2099
+ json(res, 200, { ok: true, held: true, until: new Date(until).toISOString(), staging: stagedWorkspaces ? "on" : "off" });
2100
+ }
2101
+ catch (error) {
2102
+ requestError(res, error);
2103
+ }
2104
+ }
2105
+ async function handleApi(req, res) {
2106
+ const url = new URL(req.url, `http://127.0.0.1:${port}`);
2107
+ if (url.pathname === "/healthz") {
2108
+ json(res, 200, {
2109
+ ok: true,
2110
+ identity,
2111
+ capabilities: runnerCapabilities,
2112
+ });
2113
+ return;
2114
+ }
2115
+ // Everything past the health check needs the API secret: session
2116
+ // containers reach this port through `host.docker.internal`.
2117
+ if (!isAuthorizedRunnerRequest(req, apiToken)) {
2118
+ res.setHeader("WWW-Authenticate", "Bearer");
2119
+ json(res, 401, { error: "Unauthorized" });
2120
+ return;
2121
+ }
2122
+ if (isRefusedBrowserMutation(req)) {
2123
+ json(res, 403, { error: "Forbidden" });
2124
+ return;
2125
+ }
2126
+ // Every route under /envs/ names an environment in its path; the slug
2127
+ // becomes Docker names and a host folder, so it must be a plain name.
2128
+ const envSegment = url.pathname.match(/^\/envs\/([^/]+)/);
2129
+ if (envSegment) {
2130
+ let slug = null;
2131
+ try {
2132
+ slug = decodeURIComponent(envSegment[1]);
2133
+ }
2134
+ catch {
2135
+ // Malformed escapes are refused below like any other bad slug.
2136
+ }
2137
+ if (!isValidRunnerSlug(slug)) {
2138
+ requestError(res, new RunnerSlugError());
2139
+ return;
2140
+ }
2141
+ }
2142
+ if (url.pathname === "/dev/staging-hold") {
2143
+ await handleStagingHold(req, res);
2144
+ return;
2145
+ }
2146
+ if (url.pathname === "/projects/prepare") {
2147
+ if (req.method !== "POST") {
2148
+ methodNotAllowed(res);
2149
+ return;
2150
+ }
2151
+ let request = null;
2152
+ try {
2153
+ request = parseProjectPreparationRequest(await readJson(req).catch((error) => {
2154
+ throw new ProjectPreparationRequestError(error instanceof Error ? error.message : String(error));
2155
+ }));
2156
+ json(res, 200, await prepareProject(request));
2157
+ }
2158
+ catch (error) {
2159
+ const secrets = Object.fromEntries(PREPARE_PROJECT_SECRET_NAMES.flatMap((name) => request?.envVars[name] ? [[name, request.envVars[name]]] : []));
2160
+ const message = redactEnvValues(error instanceof Error ? error.message : String(error), secrets);
2161
+ json(res, error instanceof ProjectPreparationRequestError ? 400 : 502, { ok: false, error: message });
2162
+ }
2163
+ return;
2164
+ }
2165
+ if (url.pathname === "/gadget-jobs") {
2166
+ // The build-and-test job for one app (gadget-job.ts). Like every route
2167
+ // past the health check it needs the API secret, which no container
2168
+ // has, and it is refused when it comes from a browser. The request
2169
+ // names no mount, port, image or variable: any field but the app's id,
2170
+ // a commit, files and a catalog is refused.
2171
+ if (req.method !== "POST") {
2172
+ methodNotAllowed(res);
2173
+ return;
2174
+ }
2175
+ if (!runnerCapabilities.features?.includes("gadget-job")) {
2176
+ json(res, 409, { ok: false, code: "gadget_job_unsupported", error: "This machine's image does not carry the build-and-test job. Update the machine." });
2177
+ return;
2178
+ }
2179
+ try {
2180
+ json(res, 200, await gadgetJobs.run(parseGadgetJobRequest(await readGadgetJobBody(req))));
2181
+ }
2182
+ catch (error) {
2183
+ if (error instanceof GadgetJobRequestError)
2184
+ json(res, error.status, { ok: false, code: error.code, error: error.message });
2185
+ else
2186
+ json(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) });
2187
+ }
2188
+ return;
2189
+ }
2190
+ if (url.pathname === "/gadget-map-backs") {
2191
+ // The same job's second mode: mapping an OS edit back onto an app's source, for a pull.
2192
+ // The same rules as above; the request may carry the app's id, a commit, the app's files
2193
+ // and the OS's files and nothing else.
2194
+ if (req.method !== "POST") {
2195
+ methodNotAllowed(res);
2196
+ return;
2197
+ }
2198
+ if (!runnerCapabilities.features?.includes("gadget-map-back")) {
2199
+ json(res, 409, { ok: false, code: "gadget_job_unsupported", error: `This machine's image does not map ${OS.noun} edits back. Update the machine.` });
2200
+ return;
2201
+ }
2202
+ try {
2203
+ json(res, 200, await gadgetJobs.mapBack(parseGadgetMapBackRequest(await readGadgetJobBody(req))));
2204
+ }
2205
+ catch (error) {
2206
+ if (error instanceof GadgetJobRequestError)
2207
+ json(res, error.status, { ok: false, code: error.code, error: error.message });
2208
+ else
2209
+ json(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) });
2210
+ }
2211
+ return;
2212
+ }
2213
+ const laneTicketMatch = url.pathname.match(/^\/envs\/([^/]+)\/terminal-lane-tickets$/);
2214
+ if (laneTicketMatch) {
2215
+ // Minted only for the runner's own Hub session client; the API secret
2216
+ // and the browser guard at the top of handleApi refuse everything else.
2217
+ if (req.method !== "POST") {
2218
+ methodNotAllowed(res);
2219
+ return;
2220
+ }
2221
+ if (!config.terminalLane) {
2222
+ json(res, 404, { error: "This runner offers no terminal lane" });
2223
+ return;
2224
+ }
2225
+ try {
2226
+ const request = parseTerminalLaneTicketRequest(decodeURIComponent(laneTicketMatch[1]), await readJson(req));
2227
+ const inspected = await runDocker([
2228
+ "inspect", "--type", "container", "-f", TERMINAL_LANE_INSPECT_FORMAT, containerName(request.slug),
2229
+ ]).catch(() => "");
2230
+ if (!hasRunningTerminalLane(inspected)) {
2231
+ json(res, 409, { error: "The environment has no terminal lane" });
2232
+ return;
2233
+ }
2234
+ const address = server.address();
2235
+ const listeningPort = address && typeof address === "object" ? address.port : port;
2236
+ json(res, 200, { ...terminalLaneTickets.mint(request), port: listeningPort });
2237
+ }
2238
+ catch (error) {
2239
+ json(res, 400, { error: error instanceof Error ? error.message : String(error) });
2240
+ }
2241
+ return;
2242
+ }
2243
+ if (req.method === "POST" && url.pathname === "/envs") {
2244
+ try {
2245
+ const timer = createStepTimer();
2246
+ const payload = await readJson(req);
2247
+ timer.mark("read-body");
2248
+ const createPayload = payload;
2249
+ assertValidRunnerSlug(createPayload.slug);
2250
+ await waitForInFlightStop(createPayload.slug, "create");
2251
+ await refuseWhenFull(createPayload.slug, payload);
2252
+ const record = acceptCommand(createPayload.slug, payload, "running");
2253
+ timer.mark("accept");
2254
+ const created = record.phase === "applied"
2255
+ ? await inspectAppliedRunningCommand(record, fenceStore, createPayload.envVars)
2256
+ : await scheduleCommand(record, () => executeFencedStart(record, createPayload, "create", timer));
2257
+ json(res, 201, { ...created, ...commandResponse(record) });
2258
+ }
2259
+ catch (err) {
2260
+ requestError(res, err);
2261
+ }
2262
+ return;
2263
+ }
2264
+ const inspectionMatch = url.pathname.match(/^\/envs\/([^/]+)\/inspection$/);
2265
+ if (req.method === "GET" && inspectionMatch) {
2266
+ try {
2267
+ const slug = decodeURIComponent(inspectionMatch[1]);
2268
+ // The Hub inspects before it dispatches a Start, and it may call the
2269
+ // environment stopped (the container reported its Stop) while
2270
+ // `docker stop` is still returning here. Answering then shows a
2271
+ // container still running, or the Stop's own fence write landing
2272
+ // mid-inspection (`command-changed`), and the Start is refused. So,
2273
+ // like create and start, wait briefly for the Stop to finish. A
2274
+ // container running with no Stop in flight is answered at once.
2275
+ await waitForInFlightStop(slug, "inspection", config.inspectionStopWaitMs ?? INSPECTION_STOP_WAIT_MS);
2276
+ const before = fenceStore.read(slug);
2277
+ const info = await inspectContainer(slug, true);
2278
+ const after = fenceStore.read(slug);
2279
+ const launched = fenceStore.readLaunched(slug);
2280
+ let persistence = null;
2281
+ try {
2282
+ persistence = fenceStore.readWorkspaceSyncedStop(slug);
2283
+ }
2284
+ catch {
2285
+ // An unreadable proof proves nothing; the inspection goes on without it.
2286
+ }
2287
+ json(res, 200, inspectRunnerState(slug, info ? getStatus(info) : null, before, after, failedBeforeHarnessProof(info, launched), !launched, durableStopProof(info, persistence)));
2288
+ }
2289
+ catch (error) {
2290
+ requestError(res, error, 500);
2291
+ }
2292
+ return;
2293
+ }
2294
+ const envMatch = url.pathname.match(/^\/envs\/([^/]+)$/);
2295
+ if (req.method === "GET" && envMatch) {
2296
+ const slug = decodeURIComponent(envMatch[1]);
2297
+ const info = await inspectContainer(slug);
2298
+ if (!info) {
2299
+ notFound(res);
2300
+ return;
2301
+ }
2302
+ const launched = fenceStore.readLaunched(slug);
2303
+ const preHarnessFailure = failedBeforeHarnessProof(info, launched);
2304
+ // The Hub knows this machine's pinned image, never a container's, so the
2305
+ // host is the only place a pending recreate can be detected.
2306
+ const pinnedImageId = await resolveImageId(image);
2307
+ const recreatePending = Boolean(pinnedImageId && info.Image && info.Image !== pinnedImageId);
2308
+ json(res, 200, {
2309
+ runnerId: containerName(slug),
2310
+ status: getStatus(info),
2311
+ ...(recreatePending ? { recreatePending: true } : {}),
2312
+ ...(preHarnessFailure ? { failedStartBeforeHarness: true, ...preHarnessFailure } : {}),
2313
+ });
2314
+ return;
2315
+ }
2316
+ if (req.method === "DELETE" && envMatch) {
2317
+ const slug = decodeURIComponent(envMatch[1]);
2318
+ try {
2319
+ const payload = await readJson(req).catch(() => ({}));
2320
+ const record = acceptCommand(slug, payload, "absent");
2321
+ const result = await scheduleCommand(record, () => executeFencedDestroy(record));
2322
+ json(res, 200, {
2323
+ ok: true,
2324
+ runnerId: containerName(slug),
2325
+ removed: result.removed,
2326
+ ...commandResponse(record),
2327
+ });
2328
+ }
2329
+ catch (error) {
2330
+ requestError(res, error);
2331
+ }
2332
+ return;
2333
+ }
2334
+ const idleMatch = url.pathname.match(/^\/envs\/([^/]+)\/idle-(prepare|release)$/);
2335
+ if (idleMatch) {
2336
+ if (req.method !== "POST") {
2337
+ methodNotAllowed(res);
2338
+ return;
2339
+ }
2340
+ const slug = decodeURIComponent(idleMatch[1]);
2341
+ try {
2342
+ const payload = await readJson(req).catch(() => ({}));
2343
+ if (idleMatch[2] === "release") {
2344
+ const claimId = typeof payload.claimId === "string" ? payload.claimId.trim() : "";
2345
+ if (!claimId)
2346
+ throw new Error("claimId is required");
2347
+ await releaseIdleStopInContainer(slug, claimId).catch((error) => {
2348
+ console.error(`[runner] idle claim release for ${slug} failed (it expires on its own):`, error instanceof Error ? error.message : String(error));
2349
+ });
2350
+ json(res, 200, { ok: true });
2351
+ return;
2352
+ }
2353
+ const idleTimeoutMs = Number(payload.idleTimeoutMs);
2354
+ if (!Number.isSafeInteger(idleTimeoutMs) || idleTimeoutMs <= 0)
2355
+ throw new Error("idleTimeoutMs must be a positive integer");
2356
+ const claimTtlMs = Number.isSafeInteger(payload.claimTtlMs) && Number(payload.claimTtlMs) > 0
2357
+ ? Number(payload.claimTtlMs)
2358
+ : HOST_IDLE_CLAIM_TTL_MS;
2359
+ const info = await inspectContainer(slug, true);
2360
+ if (!info || getStatus(info) !== "running") {
2361
+ json(res, 200, { ok: true, eligible: false, reason: "not_running", remainingIdleMs: idleTimeoutMs });
2362
+ return;
2363
+ }
2364
+ const startedAt = performance.now();
2365
+ const result = await prepareIdleStopInContainer(slug, idleTimeoutMs, claimTtlMs);
2366
+ console.log(`[runner] idle stop preparation for ${slug}: ${result.reason} in ${Math.round(performance.now() - startedAt)} ms`);
2367
+ json(res, 200, { ok: true, ...result });
2368
+ }
2369
+ catch (error) {
2370
+ requestError(res, error, 500);
2371
+ }
2372
+ return;
2373
+ }
2374
+ const actionMatch = url.pathname.match(/^\/envs\/([^/]+)\/(start|stop)$/);
2375
+ if (actionMatch) {
2376
+ const slug = decodeURIComponent(actionMatch[1]);
2377
+ const action = actionMatch[2];
2378
+ if (req.method !== "POST") {
2379
+ methodNotAllowed(res);
2380
+ return;
2381
+ }
2382
+ if (action === "start") {
2383
+ try {
2384
+ const payload = await readJson(req);
2385
+ const startPayload = { ...payload, slug };
2386
+ await waitForInFlightStop(slug, "start");
2387
+ await refuseWhenFull(slug, payload);
2388
+ const record = acceptCommand(slug, payload, "running");
2389
+ const started = record.phase === "applied"
2390
+ ? await inspectAppliedRunningCommand(record, fenceStore, startPayload.envVars)
2391
+ : await scheduleCommand(record, () => executeFencedStart(record, startPayload, "start"));
2392
+ json(res, 200, { ...started, ...commandResponse(record) });
2393
+ }
2394
+ catch (err) {
2395
+ requestError(res, err);
2396
+ }
2397
+ return;
2398
+ }
2399
+ if (action === "stop") {
2400
+ try {
2401
+ const payload = await readJson(req).catch(() => ({}));
2402
+ const idleClaimId = typeof payload.idleClaimId === "string" ? payload.idleClaimId.trim() : "";
2403
+ if (idleClaimId && !(await admitIdleStop(slug, payload, idleClaimId))) {
2404
+ json(res, 409, {
2405
+ error: "The idle stop was refused: the session became active or its idle claim is gone.",
2406
+ code: "idle_stop_superseded",
2407
+ });
2408
+ return;
2409
+ }
2410
+ const record = acceptCommand(slug, payload, "stopped");
2411
+ if (idleClaimId) {
2412
+ const key = commandKey(record);
2413
+ idleStopClaims.set(key, idleClaimId);
2414
+ setTimeout(() => idleStopClaims.delete(key), 10 * 60_000).unref();
2415
+ }
2416
+ const scheduledStop = scheduleFencedStop(record);
2417
+ void scheduledStop.completion.catch((error) => {
2418
+ console.error(`[runner] background fenced stop for ${slug} failed:`, error);
2419
+ });
2420
+ const acknowledgement = await scheduledStop.acknowledgement;
2421
+ fenceStore.assertCurrent(record);
2422
+ json(res, 200, {
2423
+ ok: true,
2424
+ runnerId: containerName(slug),
2425
+ callbackExpected: acknowledgement.callbackExpected,
2426
+ ...(acknowledgement.startRejectedBeforeWorkspace
2427
+ ? { startRejectedBeforeWorkspace: true }
2428
+ : {}),
2429
+ ...commandResponse(record),
2430
+ });
2431
+ }
2432
+ catch (error) {
2433
+ requestError(res, error);
2434
+ }
2435
+ return;
2436
+ }
2437
+ }
2438
+ notFound(res);
2439
+ }
2440
+ const server = http.createServer(async (req, res) => {
2441
+ try {
2442
+ await handleApi(req, res);
2443
+ }
2444
+ catch (err) {
2445
+ console.error("[runner] request failed:", err);
2446
+ json(res, 500, { error: err instanceof Error ? err.message : String(err) });
2447
+ }
2448
+ });
2449
+ server.on("upgrade", (req, socket, head) => {
2450
+ if (!handleTerminalLaneUpgrade(req, socket, head, {
2451
+ config: config.terminalLane ?? null,
2452
+ tickets: terminalLaneTickets,
2453
+ relays: terminalLaneRelays,
2454
+ openUpstream: (slug) => openDockerExecRelay(containerName(slug), {
2455
+ log: (message) => console.log(`[runner] ${slug}: ${message}`),
2456
+ }),
2457
+ log: (message) => console.log(`[runner] ${message}`),
2458
+ })) {
2459
+ socket.destroy();
2460
+ }
2461
+ });
2462
+ return new Promise((resolve, reject) => {
2463
+ server.on("error", (err) => {
2464
+ if (err.code === "EADDRINUSE") {
2465
+ reject(new Error(`Runner port ${port} is already in use — is another runner or ${CLI.command} process running?`));
2466
+ }
2467
+ else {
2468
+ reject(err);
2469
+ }
2470
+ });
2471
+ // Loopback only: the terminal lane must never be reachable from another
2472
+ // machine. Loopback is not enough on its own (containers reach it), so
2473
+ // every request but `/healthz` also needs the API secret.
2474
+ server.listen(port, "127.0.0.1", () => {
2475
+ console.log(`[runner] listening on http://127.0.0.1:${port}`);
2476
+ console.log(`[runner] image=${image}`);
2477
+ try {
2478
+ writeRunnerApiTokenFile(localStateDir, apiToken);
2479
+ }
2480
+ catch (error) {
2481
+ // Only the startup bench reads the file; the Hub session client is
2482
+ // handed the secret in process.
2483
+ console.error("[runner] could not write the API secret file:", error instanceof Error ? error.message : String(error));
2484
+ }
2485
+ console.log("[runner] relying on localhost binding, the runner API secret and this machine's hub credential");
2486
+ recoverAcceptedTerminalCommands();
2487
+ console.log(`[runner] ${stagingDecision.reason}`);
2488
+ reconcileVolumes();
2489
+ const daily = setInterval(reconcileVolumes, RUNNER_VOLUME_RECONCILE_INTERVAL_MS);
2490
+ daily.unref();
2491
+ gitMirrorTimers.push(daily);
2492
+ resolve({
2493
+ server,
2494
+ identity,
2495
+ capabilities: runnerCapabilities,
2496
+ apiToken,
2497
+ countRunningEnvironments: config.countRunningEnvironments ?? (() => countRunningEnvironments(runCapacityDocker)),
2498
+ gadgetJobLoad: () => runnerCapabilities.features?.includes("gadget-job") ? gadgetJobs.load() : null,
2499
+ close: () => {
2500
+ try {
2501
+ removeRunnerApiTokenFile(localStateDir, apiToken);
2502
+ }
2503
+ catch {
2504
+ // A stale file holds a secret no runner accepts.
2505
+ }
2506
+ for (const timer of gitMirrorTimers.splice(0))
2507
+ clearTimeout(timer);
2508
+ for (const { timer } of stageTimers.values())
2509
+ clearTimeout(timer);
2510
+ stageTimers.clear();
2511
+ for (const timer of refillTimers)
2512
+ clearTimeout(timer);
2513
+ refillTimers.clear();
2514
+ void gadgetJobs.close();
2515
+ return closeHttpServer(server);
2516
+ },
2517
+ });
2518
+ });
2519
+ });
2520
+ }