overvues 0.0.1 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -2
- package/dist/ansi.js +24 -0
- package/dist/attach-client.js +586 -0
- package/dist/auth-connect-browser.js +112 -0
- package/dist/auth-connect.js +306 -0
- package/dist/browser-bootstrap.js +190 -0
- package/dist/browser-loopback.js +289 -0
- package/dist/build-identity.js +6 -0
- package/dist/codex-reviewer-apparmor.json +4 -0
- package/dist/codex-reviewer-seccomp.json +1238 -0
- package/dist/codex-runtime-protocol.js +1 -0
- package/dist/codex-subscription.js +53 -0
- package/dist/config.js +297 -0
- package/dist/connect-comparison-code.js +26 -0
- package/dist/development-runner.js +109 -0
- package/dist/dns-doctor.js +390 -0
- package/dist/docker-names.js +47 -0
- package/dist/doctor.js +114 -0
- package/dist/gadget-job.js +804 -0
- package/dist/health-check.js +112 -0
- package/dist/help.js +71 -0
- package/dist/host-auth.js +22 -0
- package/dist/host-diagnostics.js +20 -0
- package/dist/host-runtime-identity.js +39 -0
- package/dist/host-runtime-metadata.js +56 -0
- package/dist/host-service.js +377 -0
- package/dist/host-stack.js +484 -0
- package/dist/host-supervisor.js +237 -0
- package/dist/host-update.js +387 -0
- package/dist/hub-address.js +100 -0
- package/dist/hub-client.js +459 -0
- package/dist/hub-kind.js +88 -0
- package/dist/identity.js +73 -0
- package/dist/index.js +185 -0
- package/dist/init.js +295 -0
- package/dist/isolated-run-cache.js +116 -0
- package/dist/local-stack.js +685 -0
- package/dist/machine-capacity.js +128 -0
- package/dist/managed-runner-image.js +13 -0
- package/dist/os-acted-record.js +49 -0
- package/dist/os-adapter.js +8190 -0
- package/dist/os-command.js +292 -0
- package/dist/os-handoff.js +274 -0
- package/dist/os-keychain.js +247 -0
- package/dist/os-machine.js +818 -0
- package/dist/os-owner.js +61 -0
- package/dist/os-private-file.js +98 -0
- package/dist/os-protocol.js +468 -0
- package/dist/os-session-store.js +133 -0
- package/dist/os-status.js +102 -0
- package/dist/picker.js +863 -0
- package/dist/portable-workspace-capability.js +60 -0
- package/dist/project-environment-capability.js +62 -0
- package/dist/project-environment.js +28 -0
- package/dist/project-preparation.js +134 -0
- package/dist/readiness.js +28 -0
- package/dist/redaction.js +27 -0
- package/dist/reviewer-isolation-capability.js +61 -0
- package/dist/runner-api-auth.js +80 -0
- package/dist/runner-command-fence.js +536 -0
- package/dist/runner-inspection.js +57 -0
- package/dist/runner-inventory.js +587 -0
- package/dist/runner-server.js +2520 -0
- package/dist/runner-session.js +470 -0
- package/dist/runner-slug-folder.js +39 -0
- package/dist/runner-slug.js +40 -0
- package/dist/setup.js +139 -0
- package/dist/shutdown.js +102 -0
- package/dist/splash.js +65 -0
- package/dist/staged-workspaces.js +441 -0
- package/dist/staging-disk.js +97 -0
- package/dist/terminal-lane.js +339 -0
- package/dist/terminal-recovery.js +502 -0
- package/licenses/capnweb.txt +23 -0
- package/licenses/moby-profiles.txt +212 -0
- package/licenses/ws.txt +22 -0
- package/package.json +28 -7
- package/cli.js +0 -2
|
@@ -0,0 +1,2520 @@
|
|
|
1
|
+
import http from "node:http";
|
|
2
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
3
|
+
import { HOST_PACKAGE_VERSION, readRuntimeImagePlatform } from "./host-runtime-identity.js";
|
|
4
|
+
// Moby's default syscall policy plus the namespace operations used by Bubblewrap.
|
|
5
|
+
// Provenance and Apache-2.0 license: ../licenses/moby-profiles.txt.
|
|
6
|
+
import codexReviewerSeccomp from "./codex-reviewer-seccomp.json" with { type: "json" };
|
|
7
|
+
import codexReviewerAppArmor from "./codex-reviewer-apparmor.json" with { type: "json" };
|
|
8
|
+
import { spawn } from "node:child_process";
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { tmpdir } from "node:os";
|
|
12
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
13
|
+
import { HUB_URL, LOCAL_STATE_DIR, hubControlHeaders, parseByteSize } from "./config.js";
|
|
14
|
+
import { DevelopmentHubGate, HARNESS_DIST_CONTAINER_PATH, HARNESS_OVERLAY_REFUSED, STAGING_HOLD_DEFAULT_MS, StagingHold, fetchHubBuildChannel, readDevelopmentRunnerOptions, validateHarnessOverlayDir, } from "./development-runner.js";
|
|
15
|
+
import { redactEnvValues } from "./redaction.js";
|
|
16
|
+
import { inspectRunnerState } from "./runner-inspection.js";
|
|
17
|
+
import { RunnerCommandFenceError, RunnerCommandFenceStore, RUNNER_COMMAND_PROTOCOL, } from "./runner-command-fence.js";
|
|
18
|
+
import { closeHttpServer } from "./shutdown.js";
|
|
19
|
+
import { CODEX_RUNTIME_AUTH_PROTOCOL } from "./codex-runtime-protocol.js";
|
|
20
|
+
import { validateProjectEnvironmentDelivery } from "./project-environment.js";
|
|
21
|
+
import { requireProjectEnvironmentContainer, requireProjectEnvironmentImage, resolveProjectEnvironmentImage } from "./project-environment-capability.js";
|
|
22
|
+
import { GADGET_MAP_BACK_IMAGE_LABEL, GADGET_TOOLS_IMAGE_LABEL, GadgetJobRequestError, GadgetJobs, parseGadgetJobRequest, parseGadgetMapBackRequest, readGadgetJobBody } from "./gadget-job.js";
|
|
23
|
+
import { PORTABLE_NATIVE_LABEL, requirePortableWorkspaceContainer, requirePortableWorkspaceImage, resolvePortableWorkspaceImage } from "./portable-workspace-capability.js";
|
|
24
|
+
import { parseManagedLocalRunnerImageSourceId } from "./managed-runner-image.js";
|
|
25
|
+
import { resolveHostReviewerIsolationProtocol } from "./reviewer-isolation-capability.js";
|
|
26
|
+
import { StagedWorkspaces, STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES, } from "./staged-workspaces.js";
|
|
27
|
+
import { createDockerDiskProbe, decideStagingAtStartup, stageRunDiskRefusal, } from "./staging-disk.js";
|
|
28
|
+
import { RunnerInventory, emptyDockerVolume, readRunnerInventory, reconcileRunnerVolumes, } from "./runner-inventory.js";
|
|
29
|
+
import { DOCKER_INFO_TTL_MS, TtlSingleFlight, VolumePresenceCache } from "./isolated-run-cache.js";
|
|
30
|
+
import { MIRROR_REFRESH_CONTAINER_TIMEOUT_MS, PREPARE_PROJECT_SECRET_NAMES, ProjectPreparationRequestError, buildMirrorRefreshEnv, buildMirrorRefreshRunArgs, parseMirrorRefreshOutcome, parseProjectPreparationRequest, } from "./project-preparation.js";
|
|
31
|
+
import { dockerLabel, dockerName } from "./docker-names.js";
|
|
32
|
+
import { ISOLATED_RUN_LABEL_KEY, countRunningEnvironments, isMachineFull, runCapacityDocker } from "./machine-capacity.js";
|
|
33
|
+
import { createRunnerApiToken, isAuthorizedRunnerRequest, removeRunnerApiTokenFile, writeRunnerApiTokenFile, } from "./runner-api-auth.js";
|
|
34
|
+
import { assertValidRunnerSlug, isValidRunnerSlug, RunnerSlugError } from "./runner-slug.js";
|
|
35
|
+
import { caseSafeSlugFolderName, findExactDirectoryEntry } from "./runner-slug-folder.js";
|
|
36
|
+
import { handleTerminalLaneUpgrade, hasRunningTerminalLane, openDockerExecRelay, parseTerminalLaneTicketRequest, TERMINAL_LANE_CONTAINER_HOST, TERMINAL_LANE_CONTAINER_PORT, TERMINAL_LANE_INSPECT_FORMAT, TerminalLaneRelays, TerminalLaneTickets, } from "./terminal-lane.js";
|
|
37
|
+
import { CLI, envVar, OS } from "./identity.js";
|
|
38
|
+
function containerName(slug) {
|
|
39
|
+
return dockerName(slug.replace(/[^a-zA-Z0-9_.-]/g, "-"));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Binding loopback keeps other machines out, and the API secret
|
|
43
|
+
* (`runner-api-auth.ts`) keeps out this machine's containers, which reach host
|
|
44
|
+
* loopback through `host.docker.internal` on Docker Desktop. A web page in a
|
|
45
|
+
* browser on this machine can still send a CORS "simple" request (no
|
|
46
|
+
* preflight, e.g. `Content-Type: text/plain`); it cannot know the secret, but
|
|
47
|
+
* as a second line every mutating request must also come without an Origin
|
|
48
|
+
* and as JSON, which only a non-browser client can send unprompted.
|
|
49
|
+
*/
|
|
50
|
+
function isRefusedBrowserMutation(req) {
|
|
51
|
+
if (req.method === "GET" || req.method === "HEAD")
|
|
52
|
+
return false;
|
|
53
|
+
if (req.headers.origin !== undefined)
|
|
54
|
+
return true;
|
|
55
|
+
const mediaType = String(req.headers["content-type"] ?? "").split(";")[0].trim().toLowerCase();
|
|
56
|
+
return mediaType !== "application/json";
|
|
57
|
+
}
|
|
58
|
+
function json(res, status, body) {
|
|
59
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
60
|
+
res.end(JSON.stringify(body));
|
|
61
|
+
}
|
|
62
|
+
function notFound(res) {
|
|
63
|
+
json(res, 404, { error: "Not found" });
|
|
64
|
+
}
|
|
65
|
+
function methodNotAllowed(res) {
|
|
66
|
+
json(res, 405, { error: "Method not allowed" });
|
|
67
|
+
}
|
|
68
|
+
/** A Create or Start refused before it was accepted: this machine runs as many environments as its limit allows. */
|
|
69
|
+
export class MachineFullError extends Error {
|
|
70
|
+
runningEnvironments;
|
|
71
|
+
environmentLimit;
|
|
72
|
+
status = 409;
|
|
73
|
+
code = "machine_full";
|
|
74
|
+
constructor(runningEnvironments, environmentLimit) {
|
|
75
|
+
super(`This machine is full (${runningEnvironments} of ${environmentLimit} environments running). Stop an environment or raise the limit (${envVar("MAX_ENVIRONMENTS")}).`);
|
|
76
|
+
this.runningEnvironments = runningEnvironments;
|
|
77
|
+
this.environmentLimit = environmentLimit;
|
|
78
|
+
this.name = "MachineFullError";
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function requestError(res, error, fallbackStatus = 400) {
|
|
82
|
+
if (error instanceof MachineFullError) {
|
|
83
|
+
json(res, error.status, {
|
|
84
|
+
error: error.message,
|
|
85
|
+
code: error.code,
|
|
86
|
+
runningEnvironments: error.runningEnvironments,
|
|
87
|
+
environmentLimit: error.environmentLimit,
|
|
88
|
+
});
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (error instanceof RunnerSlugError) {
|
|
92
|
+
json(res, error.status, { error: error.message, code: error.code });
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (error instanceof RunnerCommandFenceError) {
|
|
96
|
+
json(res, error.status, {
|
|
97
|
+
error: error.message,
|
|
98
|
+
code: error.code,
|
|
99
|
+
...(error.currentCommandGeneration !== undefined
|
|
100
|
+
? { currentCommandGeneration: error.currentCommandGeneration }
|
|
101
|
+
: {}),
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
json(res, fallbackStatus, { error: error instanceof Error ? error.message : String(error) });
|
|
106
|
+
}
|
|
107
|
+
function readJson(req) {
|
|
108
|
+
return new Promise((resolve, reject) => {
|
|
109
|
+
let body = "";
|
|
110
|
+
req.on("data", (chunk) => {
|
|
111
|
+
body += chunk;
|
|
112
|
+
if (body.length > 1024 * 1024) {
|
|
113
|
+
reject(new Error("Request body too large"));
|
|
114
|
+
req.destroy();
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
req.on("end", () => {
|
|
118
|
+
if (!body) {
|
|
119
|
+
resolve({});
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
resolve(JSON.parse(body));
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
reject(new Error("Invalid JSON"));
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
req.on("error", reject);
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
function runDocker(args) {
|
|
133
|
+
return new Promise((resolve, reject) => {
|
|
134
|
+
const child = spawn("docker", args, {
|
|
135
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
136
|
+
});
|
|
137
|
+
let stdout = "";
|
|
138
|
+
let stderr = "";
|
|
139
|
+
child.stdout.on("data", (chunk) => {
|
|
140
|
+
stdout += chunk.toString();
|
|
141
|
+
});
|
|
142
|
+
child.stderr.on("data", (chunk) => {
|
|
143
|
+
stderr += chunk.toString();
|
|
144
|
+
});
|
|
145
|
+
child.on("error", reject);
|
|
146
|
+
child.on("close", (code) => {
|
|
147
|
+
if (code === 0) {
|
|
148
|
+
resolve(stdout.trim());
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const error = new Error(stderr.trim() || stdout.trim() || `docker ${args.join(" ")} failed`);
|
|
152
|
+
error.code = String(code);
|
|
153
|
+
reject(error);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Whether Docker has ever started this container. Derived from Docker rather
|
|
159
|
+
* than from a flag the runner writes, so it survives a runner crash between
|
|
160
|
+
* recording a launch and starting the container.
|
|
161
|
+
*/
|
|
162
|
+
function hasRun(info) {
|
|
163
|
+
if (!info)
|
|
164
|
+
return false;
|
|
165
|
+
if (info.State?.Status === "created")
|
|
166
|
+
return false;
|
|
167
|
+
const startedAt = info.State?.StartedAt;
|
|
168
|
+
if (!startedAt)
|
|
169
|
+
return false;
|
|
170
|
+
const parsed = Date.parse(startedAt);
|
|
171
|
+
return Number.isFinite(parsed) && parsed > 0;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* A container Docker created but never started: its state is still `created`
|
|
175
|
+
* (Docker never returns a container to that state once it has run) and it has
|
|
176
|
+
* no start time. No process in it ever ran.
|
|
177
|
+
*/
|
|
178
|
+
function neverStarted(info) {
|
|
179
|
+
if (info?.State?.Status !== "created")
|
|
180
|
+
return false;
|
|
181
|
+
const startedAt = Date.parse(info.State.StartedAt ?? "");
|
|
182
|
+
return !(Number.isFinite(startedAt) && startedAt > 0);
|
|
183
|
+
}
|
|
184
|
+
function startedBefore(info, isoTimestamp) {
|
|
185
|
+
const startedAt = info?.State?.StartedAt;
|
|
186
|
+
if (!startedAt)
|
|
187
|
+
return false;
|
|
188
|
+
const started = Date.parse(startedAt);
|
|
189
|
+
const reference = Date.parse(isoTimestamp);
|
|
190
|
+
return Number.isFinite(started) && Number.isFinite(reference) && started <= reference;
|
|
191
|
+
}
|
|
192
|
+
async function inspectContainer(slug, strict = false) {
|
|
193
|
+
try {
|
|
194
|
+
// Typed: for a missing container an untyped `docker inspect` goes on to
|
|
195
|
+
// look the name up as an image, network and volume, and the volume lookup
|
|
196
|
+
// waits for any volume removal in progress (Docker's local volume driver
|
|
197
|
+
// holds one lock while it deletes a volume's files). Removing a workspace
|
|
198
|
+
// with dependencies held a fresh slug's create for 10 to 20 s.
|
|
199
|
+
const output = await runDocker(["inspect", "--type", "container", containerName(slug)]);
|
|
200
|
+
const parsed = JSON.parse(output);
|
|
201
|
+
if (strict && (!Array.isArray(parsed) || parsed.length !== 1 || typeof parsed[0]?.State?.Status !== "string" || !parsed[0].State.Status.trim())) {
|
|
202
|
+
throw new Error("Docker returned an invalid container inspection.");
|
|
203
|
+
}
|
|
204
|
+
return parsed[0] ?? null;
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
208
|
+
if (/no such (?:object|container)/i.test(message)) {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
throw err;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
function getStatus(info) {
|
|
215
|
+
const status = info?.State?.Status;
|
|
216
|
+
if (!status)
|
|
217
|
+
return "stopped";
|
|
218
|
+
if (status === "exited")
|
|
219
|
+
return "stopped";
|
|
220
|
+
return status;
|
|
221
|
+
}
|
|
222
|
+
async function forceRemoveContainer(slug) {
|
|
223
|
+
try {
|
|
224
|
+
await runDocker(["rm", "-f", containerName(slug)]);
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
catch (err) {
|
|
228
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229
|
+
if (message.includes("No such container")) {
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
throw err;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
async function removeVolumeIfPresent(volume, image) {
|
|
236
|
+
// Emptied from a container first, so the removal does not hold Docker's
|
|
237
|
+
// volume lock (and with it every other create) for the length of the delete.
|
|
238
|
+
await emptyDockerVolume(runDocker, image, volume);
|
|
239
|
+
try {
|
|
240
|
+
await runDocker(["volume", "rm", volume]);
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
catch (err) {
|
|
244
|
+
const message = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
245
|
+
if (message.includes("no such volume"))
|
|
246
|
+
return false;
|
|
247
|
+
throw err;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Removes the slug's restore cache, the workspace volume it recorded (an
|
|
252
|
+
* adopted staged volume keeps its own name) and the default-named one, which
|
|
253
|
+
* an older runner may have used before the record existed.
|
|
254
|
+
*/
|
|
255
|
+
async function removeWorkspaceVolumes(slug, recorded, image) {
|
|
256
|
+
await removeVolumeIfPresent(`${RESTORE_CACHE_VOLUME_PREFIX}${slug}`, image);
|
|
257
|
+
const fallback = workspaceVolumeName(slug);
|
|
258
|
+
let removed = false;
|
|
259
|
+
if (recorded && recorded !== fallback)
|
|
260
|
+
removed = await removeVolumeIfPresent(recorded, image);
|
|
261
|
+
return (await removeVolumeIfPresent(fallback, image)) || removed;
|
|
262
|
+
}
|
|
263
|
+
async function removeStoppedContainer(slug) {
|
|
264
|
+
try {
|
|
265
|
+
await runDocker(["rm", containerName(slug)]);
|
|
266
|
+
return true;
|
|
267
|
+
}
|
|
268
|
+
catch (err) {
|
|
269
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
270
|
+
if (message.includes("No such container"))
|
|
271
|
+
return false;
|
|
272
|
+
throw err;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
export const HOST_OPENCODE_STATE_MOUNT_PATH = "/home/tiller/.local/share/opencode";
|
|
276
|
+
const START_OP_ID_ENV_VAR = "TILLER_LIFECYCLE_START_OP_ID";
|
|
277
|
+
const HOST_COMMAND_GENERATION_ENV_VAR = "TILLER_HOST_COMMAND_GENERATION";
|
|
278
|
+
const HOST_COMMAND_FENCE_REQUIRED_ENV_VAR = "TILLER_HOST_COMMAND_FENCE_REQUIRED";
|
|
279
|
+
const HOST_COMMAND_FENCE_CONTAINER_PATH = "/run/tiller-host-command";
|
|
280
|
+
const HOST_COMMAND_GENERATION_LABEL = dockerLabel("command-generation");
|
|
281
|
+
const HOST_OPERATION_ID_LABEL = dockerLabel("operation-id");
|
|
282
|
+
const HOST_CONFIG_HASH_LABEL = dockerLabel("config-hash");
|
|
283
|
+
const RESTORE_CACHE_VOLUME_PREFIX = dockerName("restore-");
|
|
284
|
+
const WORKSPACE_VOLUME_PREFIX = dockerName("ws-");
|
|
285
|
+
const WORKSPACE_CONTAINER_PATH = "/workspace";
|
|
286
|
+
const PACKAGE_CACHE_ROOT = "/var/cache/tiller";
|
|
287
|
+
// Shared across every environment container on this machine. These hold public
|
|
288
|
+
// package archives, never workspace or credential state, and they live outside
|
|
289
|
+
// /home/tiller/.cache because the OpenCode seed step chowns that tree on every
|
|
290
|
+
// boot and would traverse them.
|
|
291
|
+
const PACKAGE_CACHE_MOUNTS = [
|
|
292
|
+
{
|
|
293
|
+
volume: dockerName("cache-npm"),
|
|
294
|
+
target: `${PACKAGE_CACHE_ROOT}/npm`,
|
|
295
|
+
env: { npm_config_cache: `${PACKAGE_CACHE_ROOT}/npm` },
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
volume: dockerName("cache-pnpm"),
|
|
299
|
+
target: `${PACKAGE_CACHE_ROOT}/pnpm`,
|
|
300
|
+
env: { npm_config_store_dir: `${PACKAGE_CACHE_ROOT}/pnpm` },
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
volume: dockerName("cache-yarn"),
|
|
304
|
+
target: `${PACKAGE_CACHE_ROOT}/yarn`,
|
|
305
|
+
env: { YARN_CACHE_FOLDER: `${PACKAGE_CACHE_ROOT}/yarn` },
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
volume: dockerName("cache-cargo"),
|
|
309
|
+
target: `${PACKAGE_CACHE_ROOT}/cargo`,
|
|
310
|
+
env: { CARGO_HOME: `${PACKAGE_CACHE_ROOT}/cargo` },
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
volume: dockerName("cache-go"),
|
|
314
|
+
target: `${PACKAGE_CACHE_ROOT}/go`,
|
|
315
|
+
env: { GOMODCACHE: `${PACKAGE_CACHE_ROOT}/go` },
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
volume: dockerName("cache-pip"),
|
|
319
|
+
target: `${PACKAGE_CACHE_ROOT}/pip`,
|
|
320
|
+
env: { PIP_CACHE_DIR: `${PACKAGE_CACHE_ROOT}/pip` },
|
|
321
|
+
},
|
|
322
|
+
];
|
|
323
|
+
export function workspaceVolumeName(slug) {
|
|
324
|
+
return `${WORKSPACE_VOLUME_PREFIX}${slug.replace(/[^a-zA-Z0-9_.-]/g, "-")}`;
|
|
325
|
+
}
|
|
326
|
+
// The image creates this directory owned by tiller-mirror (uid 1002) after the
|
|
327
|
+
// cache chown, so a new mirror volume is writable by that account only: the
|
|
328
|
+
// privileged bootstrap fills it as tiller-mirror and every workload reads it.
|
|
329
|
+
export const GIT_MIRROR_CONTAINER_PATH = `${PACKAGE_CACHE_ROOT}/git-mirror`;
|
|
330
|
+
const GIT_MIRROR_VOLUME_PREFIX = dockerName("mirror-");
|
|
331
|
+
const GIT_MIRROR_ENV_VAR = "TILLER_GIT_MIRROR_PATH";
|
|
332
|
+
export { GIT_MIRROR_IDLE_LIMIT_MS } from "./runner-inventory.js";
|
|
333
|
+
/** The canonical lowercase `https://github.com/<owner>/<repo>.git`, or null. */
|
|
334
|
+
export function canonicalGitHubRepositoryUrl(repoUrl) {
|
|
335
|
+
// Same address rule as the harness's frozenCheckoutRepository.
|
|
336
|
+
const match = /^https:\/\/github\.com\/([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?\/([a-z0-9._-]+?))(?:\.git)?$/i.exec(repoUrl);
|
|
337
|
+
if (!match || [".", ".."].includes(match[2]))
|
|
338
|
+
return null;
|
|
339
|
+
return `https://github.com/${match[1].toLowerCase()}.git`;
|
|
340
|
+
}
|
|
341
|
+
/** One mirror volume per repository on this machine; null for non-GitHub repositories. */
|
|
342
|
+
export function mirrorVolumeName(repoUrl) {
|
|
343
|
+
const canonical = canonicalGitHubRepositoryUrl(repoUrl);
|
|
344
|
+
if (!canonical)
|
|
345
|
+
return null;
|
|
346
|
+
return `${GIT_MIRROR_VOLUME_PREFIX}${createHash("sha256").update(canonical).digest("hex").slice(0, 16)}`;
|
|
347
|
+
}
|
|
348
|
+
/** The mirrors in the runner inventory, by volume. */
|
|
349
|
+
export function readGitMirrorRecords(localStateDir = LOCAL_STATE_DIR) {
|
|
350
|
+
return Object.fromEntries(Object.entries(readRunnerInventory(localStateDir)).flatMap(([volume, entry]) => entry.kind === "mirror" && entry.repoUrl ? [[volume, { repoUrl: entry.repoUrl, lastUsedAt: entry.lastUsedAt }]] : []));
|
|
351
|
+
}
|
|
352
|
+
export function recordGitMirrorUse(volume, repoUrl, localStateDir = LOCAL_STATE_DIR, now = Date.now()) {
|
|
353
|
+
new RunnerInventory(localStateDir, () => now).recordMirrorUse(volume, repoUrl);
|
|
354
|
+
}
|
|
355
|
+
async function volumeExists(volume) {
|
|
356
|
+
return gitMirrorVolumeExists(volume);
|
|
357
|
+
}
|
|
358
|
+
async function gitMirrorVolumeExists(volume) {
|
|
359
|
+
try {
|
|
360
|
+
await runDocker(["volume", "inspect", volume]);
|
|
361
|
+
return true;
|
|
362
|
+
}
|
|
363
|
+
catch {
|
|
364
|
+
return false;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* The mirror a container should mount, if any. Environments mount it
|
|
369
|
+
* read-write and fill it from their privileged bootstrap. One-shot runs only
|
|
370
|
+
* read it, and only when it already exists, so they never create an empty one
|
|
371
|
+
* (with `mirrors`, as far as the runner knows; see `VolumePresenceCache`).
|
|
372
|
+
*/
|
|
373
|
+
export async function resolveGitMirrorMount(payload, localStateDir = LOCAL_STATE_DIR, mirrors) {
|
|
374
|
+
const canonical = canonicalGitHubRepositoryUrl(payload.repoUrl);
|
|
375
|
+
const volume = canonical ? mirrorVolumeName(canonical) : null;
|
|
376
|
+
if (!canonical || !volume)
|
|
377
|
+
return null;
|
|
378
|
+
const readOnly = isIsolatedRun(payload.envVars);
|
|
379
|
+
if (readOnly && !(await (mirrors ? mirrors.exists(volume) : gitMirrorVolumeExists(volume))))
|
|
380
|
+
return null;
|
|
381
|
+
try {
|
|
382
|
+
recordGitMirrorUse(volume, canonical, localStateDir);
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
console.error(`[runner] could not record Git mirror use for ${volume}:`, error instanceof Error ? error.message : String(error));
|
|
386
|
+
}
|
|
387
|
+
return { volume, readOnly };
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* One-shot runs materialize their own checkout and must not inherit another
|
|
391
|
+
* run's workspace, so they get no persistent volume and mount the shared
|
|
392
|
+
* caches read-only.
|
|
393
|
+
*/
|
|
394
|
+
function isIsolatedRun(envVars) {
|
|
395
|
+
const mode = envVars.TILLER_BOOTSTRAP_MODE;
|
|
396
|
+
return mode === "plan-writer"
|
|
397
|
+
|| mode === "planner-run"
|
|
398
|
+
|| mode === "env-review-run"
|
|
399
|
+
|| mode === "github-env-publish"
|
|
400
|
+
|| mode === "scm-land";
|
|
401
|
+
}
|
|
402
|
+
export const ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE = 75;
|
|
403
|
+
const ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE = 76;
|
|
404
|
+
/** The image label that says its land job knows the two modes of an app's Ship, `prepare` and `advance`. */
|
|
405
|
+
export const SCM_LAND_MODES_IMAGE_LABEL = "dev.tiller.scm-land-prepare-advance";
|
|
406
|
+
function resolveRunnerFeatures(flags) {
|
|
407
|
+
// `command-receipts`: the Hub session client acknowledges each runner
|
|
408
|
+
// command as soon as it arrives (runner-session.ts), before running it.
|
|
409
|
+
const features = ["git-mirror", "prepare-mirror-commit", "prepare-project", "idle-stop", "command-receipts"];
|
|
410
|
+
if (flags.stagedWorkspaces)
|
|
411
|
+
features.push("staged-workspaces");
|
|
412
|
+
if (flags.terminalLane)
|
|
413
|
+
features.push("terminal-lane");
|
|
414
|
+
// `gadget-job`: the image carries the gadget tools and the build-and-test job (gadget-job.ts).
|
|
415
|
+
if (flags.gadgetJob)
|
|
416
|
+
features.push("gadget-job");
|
|
417
|
+
// `gadget-map-back`: that job also maps an OS edit back onto the source, for a pull.
|
|
418
|
+
if (flags.gadgetJob && flags.gadgetMapBack)
|
|
419
|
+
features.push("gadget-map-back");
|
|
420
|
+
// `scm-land-modes`: the image's land job can stop before main (`prepare`) and move main later
|
|
421
|
+
// (`advance`), which the Ship of an app needs. An older image ignores the mode and lands main.
|
|
422
|
+
if (flags.scmLandModes)
|
|
423
|
+
features.push("scm-land-modes");
|
|
424
|
+
return features.sort();
|
|
425
|
+
}
|
|
426
|
+
function resolveRunnerCapabilities(image, flags = { stagedWorkspaces: false }) {
|
|
427
|
+
const reviewerIsolationProtocol = resolveHostReviewerIsolationProtocol(image, parseManagedLocalRunnerImageSourceId(image));
|
|
428
|
+
const features = resolveRunnerFeatures({
|
|
429
|
+
...flags,
|
|
430
|
+
gadgetJob: Boolean(resolvePortableWorkspaceImage(image, GADGET_TOOLS_IMAGE_LABEL)),
|
|
431
|
+
gadgetMapBack: Boolean(resolvePortableWorkspaceImage(image, GADGET_MAP_BACK_IMAGE_LABEL)),
|
|
432
|
+
scmLandModes: Boolean(resolvePortableWorkspaceImage(image, SCM_LAND_MODES_IMAGE_LABEL)),
|
|
433
|
+
});
|
|
434
|
+
return {
|
|
435
|
+
runnerCommandProtocol: RUNNER_COMMAND_PROTOCOL,
|
|
436
|
+
codexRuntimeAuthProtocol: CODEX_RUNTIME_AUTH_PROTOCOL,
|
|
437
|
+
...(reviewerIsolationProtocol ? { reviewerIsolationProtocol } : {}),
|
|
438
|
+
...(resolveProjectEnvironmentImage(image) ? { projectEnvironmentProtocol: 1 } : {}),
|
|
439
|
+
...(resolvePortableWorkspaceImage(image) ? { portableWorkspaceRestoreProtocol: 1 } : {}),
|
|
440
|
+
...(resolvePortableWorkspaceImage(image, PORTABLE_NATIVE_LABEL) ? { portableNativeRestoreProtocol: 1 } : {}),
|
|
441
|
+
...(features.length ? { features: Object.freeze(features) } : {}),
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
function redactError(error, envVars) {
|
|
445
|
+
return new Error(redactEnvValues(error instanceof Error ? error.message : String(error), envVars));
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* The folder name that holds one slug's OpenCode state: the lowercased slug,
|
|
449
|
+
* `+`, and 32 hex digits of the slug's SHA-256 (`runner-slug-folder.ts`).
|
|
450
|
+
* macOS disks ignore case by default, so the raw slug would give `Foo` and
|
|
451
|
+
* `foo` one folder while their containers and volumes stay apart; the hash of
|
|
452
|
+
* the exact slug keeps them apart here too.
|
|
453
|
+
*/
|
|
454
|
+
export function opencodeStateFolderName(slug) {
|
|
455
|
+
return caseSafeSlugFolderName(slug);
|
|
456
|
+
}
|
|
457
|
+
/** The name runners from 2026-11-12 gave new folders: 16 hex digits instead of 32. */
|
|
458
|
+
export function legacyOpencodeStateFolderName(slug) {
|
|
459
|
+
return caseSafeSlugFolderName(slug, 16);
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* The folder (directly inside `<state dir>/opencode`) mounted read-write into
|
|
463
|
+
* an OpenCode environment and deleted when it is destroyed.
|
|
464
|
+
*
|
|
465
|
+
* Earlier runners named the folder after the raw slug, and then after the
|
|
466
|
+
* slug with 16 hex digits of its hash. Such a folder is used in place when a
|
|
467
|
+
* directory entry with exactly that name exists, in that order: the entry's
|
|
468
|
+
* own name is compared, because on a case-insensitive disk `Foo` would also
|
|
469
|
+
* "exist" when only `foo` does. It is not moved: a running environment may
|
|
470
|
+
* have it mounted. Every other slug gets `opencodeStateFolderName`.
|
|
471
|
+
*
|
|
472
|
+
* The API refuses slugs that are not plain names (`runner-slug.ts`); as a
|
|
473
|
+
* second safeguard this refuses any path not directly inside the `opencode`
|
|
474
|
+
* folder.
|
|
475
|
+
*/
|
|
476
|
+
export function resolveHostOpencodeStatePath(slug, localStateDir = LOCAL_STATE_DIR) {
|
|
477
|
+
const root = resolve(localStateDir, "opencode");
|
|
478
|
+
const refuse = () => new RunnerSlugError(`Refusing an OpenCode state folder outside ${root}.`);
|
|
479
|
+
if (!isValidRunnerSlug(slug))
|
|
480
|
+
throw refuse();
|
|
481
|
+
const folder = findExactDirectoryEntry(root, [slug, legacyOpencodeStateFolderName(slug)])
|
|
482
|
+
?? opencodeStateFolderName(slug);
|
|
483
|
+
const path = resolve(root, folder);
|
|
484
|
+
if (dirname(path) !== root || basename(path) !== folder)
|
|
485
|
+
throw refuse();
|
|
486
|
+
return path;
|
|
487
|
+
}
|
|
488
|
+
export function resolveOpencodeMount(payload, localStateDir = LOCAL_STATE_DIR) {
|
|
489
|
+
if (payload.envVars.TILLER_HARNESS !== "opencode"
|
|
490
|
+
|| payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE !== undefined
|
|
491
|
+
|| payload.envVars.TILLER_PORTABLE_NATIVE_RESTORE !== undefined) {
|
|
492
|
+
return null;
|
|
493
|
+
}
|
|
494
|
+
const sourcePath = resolveHostOpencodeStatePath(payload.slug, localStateDir);
|
|
495
|
+
mkdirSync(sourcePath, { recursive: true, mode: 0o700 });
|
|
496
|
+
return {
|
|
497
|
+
sourcePath,
|
|
498
|
+
targetPath: HOST_OPENCODE_STATE_MOUNT_PATH,
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
export function removeLocalOpencodeState(slug, localStateDir = LOCAL_STATE_DIR) {
|
|
502
|
+
const sourcePath = resolveHostOpencodeStatePath(slug, localStateDir);
|
|
503
|
+
if (!existsSync(sourcePath)) {
|
|
504
|
+
return false;
|
|
505
|
+
}
|
|
506
|
+
rmSync(sourcePath, { recursive: true, force: true });
|
|
507
|
+
return true;
|
|
508
|
+
}
|
|
509
|
+
function isPermissionError(error) {
|
|
510
|
+
const code = error?.code;
|
|
511
|
+
return code === "EACCES" || code === "EPERM";
|
|
512
|
+
}
|
|
513
|
+
async function removeLocalOpencodeStateForRunner(slug, image) {
|
|
514
|
+
const sourcePath = resolveHostOpencodeStatePath(slug);
|
|
515
|
+
if (!existsSync(sourcePath)) {
|
|
516
|
+
return false;
|
|
517
|
+
}
|
|
518
|
+
try {
|
|
519
|
+
rmSync(sourcePath, { recursive: true, force: true });
|
|
520
|
+
return true;
|
|
521
|
+
}
|
|
522
|
+
catch (error) {
|
|
523
|
+
if (!isPermissionError(error)) {
|
|
524
|
+
throw error;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
await runDocker([
|
|
528
|
+
"run",
|
|
529
|
+
"--rm",
|
|
530
|
+
"--entrypoint",
|
|
531
|
+
"sh",
|
|
532
|
+
"-v",
|
|
533
|
+
`${dirname(sourcePath)}:/tiller-opencode-state`,
|
|
534
|
+
image,
|
|
535
|
+
"-lc",
|
|
536
|
+
'rm -rf -- "$1"',
|
|
537
|
+
"sh",
|
|
538
|
+
`/tiller-opencode-state/${basename(sourcePath)}`,
|
|
539
|
+
]);
|
|
540
|
+
return true;
|
|
541
|
+
}
|
|
542
|
+
export function buildContainerEnvVars(payload, options) {
|
|
543
|
+
const packageCacheEnv = {};
|
|
544
|
+
for (const mount of PACKAGE_CACHE_MOUNTS) {
|
|
545
|
+
Object.assign(packageCacheEnv, mount.env);
|
|
546
|
+
}
|
|
547
|
+
return {
|
|
548
|
+
// Point the package managers at the shared caches first so a repository's
|
|
549
|
+
// own values, and the Hub's, still win.
|
|
550
|
+
...packageCacheEnv,
|
|
551
|
+
...payload.envVars,
|
|
552
|
+
...(options?.gitMirror ? { [GIT_MIRROR_ENV_VAR]: GIT_MIRROR_CONTAINER_PATH } : {}),
|
|
553
|
+
TILLER_STOP_CONTROL_SECRET: randomBytes(32).toString("hex"),
|
|
554
|
+
// A session's harness serves the local terminal lane on this port, on the
|
|
555
|
+
// container's own loopback; the runner reaches it through `docker exec`.
|
|
556
|
+
// Isolated runs (planners, reviews) have no interactive terminal.
|
|
557
|
+
...(options?.terminalLane && !isIsolatedRun(payload.envVars)
|
|
558
|
+
? {
|
|
559
|
+
TILLER_TERMINAL_LANE_PORT: String(TERMINAL_LANE_CONTAINER_PORT),
|
|
560
|
+
TILLER_TERMINAL_LANE_HOST: TERMINAL_LANE_CONTAINER_HOST,
|
|
561
|
+
TILLER_TERMINAL_LANE_MACHINE_ID: options.terminalLane.machineId,
|
|
562
|
+
}
|
|
563
|
+
: {}),
|
|
564
|
+
...(payload.operationId?.trim() ? { [START_OP_ID_ENV_VAR]: payload.operationId.trim() } : {}),
|
|
565
|
+
...(Number.isSafeInteger(payload.commandGeneration) && (payload.commandGeneration ?? 0) > 0
|
|
566
|
+
? {
|
|
567
|
+
[HOST_COMMAND_GENERATION_ENV_VAR]: String(payload.commandGeneration),
|
|
568
|
+
[HOST_COMMAND_FENCE_REQUIRED_ENV_VAR]: "1",
|
|
569
|
+
}
|
|
570
|
+
: {}),
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
export function buildDockerEnvFileContent(envVars) {
|
|
574
|
+
validateProjectEnvironmentDelivery(envVars);
|
|
575
|
+
return Object.entries(envVars).map(([key, value]) => {
|
|
576
|
+
if (value.includes("\0") || value.includes("\n") || value.includes("\r")) {
|
|
577
|
+
throw new Error(`${key} cannot be written to a Docker env file.`);
|
|
578
|
+
}
|
|
579
|
+
return `${key}=${value}`;
|
|
580
|
+
}).join("\n") + "\n";
|
|
581
|
+
}
|
|
582
|
+
export function buildHostLaunchFileContent(envVars) {
|
|
583
|
+
buildDockerEnvFileContent(envVars);
|
|
584
|
+
// The entrypoint sources this file; Docker's literal env-file syntax is not shell syntax.
|
|
585
|
+
// Stop clients launched by docker exec inherit the container's original credential.
|
|
586
|
+
// Retain that credential across warm starts instead of overriding only the server.
|
|
587
|
+
return Object.entries(envVars).filter(([key]) => key !== "TILLER_STOP_CONTROL_SECRET").map(([key, value]) => {
|
|
588
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
|
|
589
|
+
throw new Error("Invalid host launch environment key.");
|
|
590
|
+
return `${key}='${value.replaceAll("'", "'\\''")}'`;
|
|
591
|
+
}).join("\n") + "\n";
|
|
592
|
+
}
|
|
593
|
+
function usesCodexReviewerSandbox(envVars) {
|
|
594
|
+
return ["planner-run", "env-review-run"].includes(envVars.TILLER_BOOTSTRAP_MODE) && envVars.TILLER_HARNESS === "codex";
|
|
595
|
+
}
|
|
596
|
+
export function createDockerEnvFile(envVars) {
|
|
597
|
+
let dirPath = null;
|
|
598
|
+
try {
|
|
599
|
+
dirPath = mkdtempSync(join(tmpdir(), "tiller-env-"));
|
|
600
|
+
chmodSync(dirPath, 0o700);
|
|
601
|
+
const filePath = join(dirPath, "env");
|
|
602
|
+
writeFileSync(filePath, buildDockerEnvFileContent(envVars), { encoding: "utf8", mode: 0o600 });
|
|
603
|
+
chmodSync(filePath, 0o600);
|
|
604
|
+
if (usesCodexReviewerSandbox(envVars)) {
|
|
605
|
+
// The parser helper drops DAC_OVERRIDE; this policy is public data inside
|
|
606
|
+
// the private temporary directory, so it can be world-readable.
|
|
607
|
+
writeFileSync(join(dirPath, "codex-reviewer.apparmor"), codexReviewerAppArmor.profile, { encoding: "utf8", mode: 0o644 });
|
|
608
|
+
writeFileSync(join(dirPath, "codex-reviewer-seccomp.json"), JSON.stringify(codexReviewerSeccomp), {
|
|
609
|
+
encoding: "utf8",
|
|
610
|
+
mode: 0o600,
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
const cleanupPath = dirPath;
|
|
614
|
+
return {
|
|
615
|
+
filePath,
|
|
616
|
+
cleanup: () => rmSync(cleanupPath, { recursive: true, force: true }),
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
catch (error) {
|
|
620
|
+
if (dirPath) {
|
|
621
|
+
rmSync(dirPath, { recursive: true, force: true });
|
|
622
|
+
}
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
export function buildContainerRunArgs(payload, image, envFilePath, options) {
|
|
627
|
+
const name = containerName(payload.slug);
|
|
628
|
+
// `create`, not `run -d`: the container id must be known before any process
|
|
629
|
+
// runs so the launch record can name it and the entrypoint can verify that
|
|
630
|
+
// the record is its own.
|
|
631
|
+
const args = [
|
|
632
|
+
"create",
|
|
633
|
+
...(image.startsWith("sha256:") ? ["--pull=never"] : []),
|
|
634
|
+
"--name",
|
|
635
|
+
name,
|
|
636
|
+
"--label",
|
|
637
|
+
`${dockerLabel("slug")}=${payload.slug}`,
|
|
638
|
+
"--add-host",
|
|
639
|
+
"host.docker.internal:host-gateway",
|
|
640
|
+
];
|
|
641
|
+
if (options?.configHash) {
|
|
642
|
+
args.push("--label", `${HOST_CONFIG_HASH_LABEL}=${options.configHash}`);
|
|
643
|
+
}
|
|
644
|
+
if (payload.commandGeneration && options?.fenceDirectory) {
|
|
645
|
+
args.push("--label", `${HOST_COMMAND_GENERATION_LABEL}=${payload.commandGeneration}`, "--label", `${HOST_OPERATION_ID_LABEL}=${payload.operationId}`, "-v", `${options.fenceDirectory}:${HOST_COMMAND_FENCE_CONTAINER_PATH}:ro`);
|
|
646
|
+
}
|
|
647
|
+
const isolated = isIsolatedRun(payload.envVars);
|
|
648
|
+
if (isolated) {
|
|
649
|
+
// Left out of the environment count and the limit (`machine-capacity.ts`).
|
|
650
|
+
args.push("--label", `${dockerLabel(ISOLATED_RUN_LABEL_KEY)}=true`);
|
|
651
|
+
}
|
|
652
|
+
else {
|
|
653
|
+
// The workspace outlives the container so a recreate keeps the checkout and
|
|
654
|
+
// its installed dependencies. The Hub remains its durable owner; this is a
|
|
655
|
+
// cache of the Hub's copy.
|
|
656
|
+
// Archive recovery needs an empty destination. Leave the retained volume
|
|
657
|
+
// untouched and restore into the fresh container's writable filesystem.
|
|
658
|
+
if (payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined)
|
|
659
|
+
args.push("-v", `${options?.workspaceVolume ?? workspaceVolumeName(payload.slug)}:${WORKSPACE_CONTAINER_PATH}`);
|
|
660
|
+
args.push("-v", `${RESTORE_CACHE_VOLUME_PREFIX}${payload.slug}:/var/lib/tiller-restore-cache`);
|
|
661
|
+
// No port is published: the terminal lane is reached through `docker exec`
|
|
662
|
+
// (`terminal-lane.ts`), so nothing on this machine but the runner reaches it.
|
|
663
|
+
}
|
|
664
|
+
for (const mount of PACKAGE_CACHE_MOUNTS) {
|
|
665
|
+
args.push("-v", `${mount.volume}:${mount.target}${isolated ? ":ro" : ""}`);
|
|
666
|
+
}
|
|
667
|
+
if (options?.gitMirror) {
|
|
668
|
+
args.push("-v", `${options.gitMirror.volume}:${GIT_MIRROR_CONTAINER_PATH}${options.gitMirror.readOnly ? ":ro" : ""}`);
|
|
669
|
+
}
|
|
670
|
+
if (options?.harnessOverlayDir) {
|
|
671
|
+
args.push("-v", `${options.harnessOverlayDir}:${HARNESS_DIST_CONTAINER_PATH}:ro`);
|
|
672
|
+
}
|
|
673
|
+
const opencodeMount = resolveOpencodeMount(payload);
|
|
674
|
+
if (opencodeMount) {
|
|
675
|
+
args.push("-v", `${opencodeMount.sourcePath}:${opencodeMount.targetPath}`);
|
|
676
|
+
}
|
|
677
|
+
if (usesCodexReviewerSandbox(payload.envVars)) {
|
|
678
|
+
args.push("--security-opt", `seccomp=${join(dirname(envFilePath), "codex-reviewer-seccomp.json")}`);
|
|
679
|
+
if (options?.codexReviewerAppArmorProfile) {
|
|
680
|
+
args.push("--security-opt", `apparmor=${options.codexReviewerAppArmorProfile}`);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
args.push("--env-file", envFilePath, image);
|
|
684
|
+
return args;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Identifies the shape of a container: everything `docker create` was given
|
|
688
|
+
* except the per-start values, which a reused container receives through its
|
|
689
|
+
* launch file instead. Paths under the temporary env directory are replaced by
|
|
690
|
+
* a sentinel because that directory is new on every start.
|
|
691
|
+
*/
|
|
692
|
+
export function computeContainerConfigHash(payload, image, envFilePath, options) {
|
|
693
|
+
const args = buildContainerRunArgs(payload, image, envFilePath, options);
|
|
694
|
+
const envDirectory = dirname(envFilePath);
|
|
695
|
+
const normalized = args.map((arg) => {
|
|
696
|
+
// The generation and operation id identify a start, not a container shape.
|
|
697
|
+
// A reused container keeps the labels of the start that created it, so they
|
|
698
|
+
// must not take part in the comparison.
|
|
699
|
+
if (arg.startsWith(`${HOST_COMMAND_GENERATION_LABEL}=`)) {
|
|
700
|
+
return `${HOST_COMMAND_GENERATION_LABEL}=<generation>`;
|
|
701
|
+
}
|
|
702
|
+
if (arg.startsWith(`${HOST_OPERATION_ID_LABEL}=`)) {
|
|
703
|
+
return `${HOST_OPERATION_ID_LABEL}=<operation-id>`;
|
|
704
|
+
}
|
|
705
|
+
return arg.split(envDirectory).join("<env-dir>");
|
|
706
|
+
});
|
|
707
|
+
return createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
|
|
708
|
+
}
|
|
709
|
+
function containerConfigHash(info) {
|
|
710
|
+
return info?.Config?.Labels?.[HOST_CONFIG_HASH_LABEL]?.trim() || null;
|
|
711
|
+
}
|
|
712
|
+
async function resolveImageId(image) {
|
|
713
|
+
try {
|
|
714
|
+
const output = await runDocker(["image", "inspect", image, "--format", "{{.Id}}"]);
|
|
715
|
+
return output.trim() || null;
|
|
716
|
+
}
|
|
717
|
+
catch {
|
|
718
|
+
return null;
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
// Install only Tiller's named container policy; never change host-wide userns
|
|
722
|
+
// settings. The trusted parser helper has MAC_ADMIN solely to load this policy.
|
|
723
|
+
export function buildReviewerAppArmorInstallArgs(image, profilePath) {
|
|
724
|
+
return [
|
|
725
|
+
"run", "--rm", "--network", "none", "--cap-drop", "ALL", "--cap-add", "MAC_ADMIN",
|
|
726
|
+
"--security-opt", "apparmor=unconfined",
|
|
727
|
+
"--mount", "type=bind,src=/sys/kernel/security,dst=/sys/kernel/security",
|
|
728
|
+
"--mount", `type=bind,src=${profilePath},dst=/tmp/tiller-reviewer.apparmor,readonly`,
|
|
729
|
+
"--entrypoint", "/usr/sbin/apparmor_parser", image,
|
|
730
|
+
"--replace", "--skip-cache", "/tmp/tiller-reviewer.apparmor",
|
|
731
|
+
];
|
|
732
|
+
}
|
|
733
|
+
const reviewerAppArmorInstalls = new Map();
|
|
734
|
+
/** `docker info` once a minute at most; concurrent reviewer creates share one call. */
|
|
735
|
+
export const dockerSecurityOptions = new TtlSingleFlight(async () => {
|
|
736
|
+
const securityOptions = JSON.parse(await runDocker(["info", "--format", "{{json .SecurityOptions}}"]));
|
|
737
|
+
if (!Array.isArray(securityOptions))
|
|
738
|
+
throw new Error("Docker did not report its security options.");
|
|
739
|
+
return securityOptions;
|
|
740
|
+
}, DOCKER_INFO_TTL_MS);
|
|
741
|
+
export async function prepareReviewerAppArmor(image, envFilePath) {
|
|
742
|
+
const securityOptions = await dockerSecurityOptions.get();
|
|
743
|
+
if (!securityOptions.some((option) => typeof option === "string" && option.startsWith("name=apparmor")))
|
|
744
|
+
return undefined;
|
|
745
|
+
let install = reviewerAppArmorInstalls.get(image);
|
|
746
|
+
if (!install) {
|
|
747
|
+
install = runDocker(buildReviewerAppArmorInstallArgs(image, join(dirname(envFilePath), "codex-reviewer.apparmor")))
|
|
748
|
+
.then(() => undefined)
|
|
749
|
+
.catch((error) => {
|
|
750
|
+
reviewerAppArmorInstalls.delete(image);
|
|
751
|
+
throw new Error(`Could not install the Codex reviewer AppArmor profile: ${error instanceof Error ? error.message : String(error)}`);
|
|
752
|
+
});
|
|
753
|
+
reviewerAppArmorInstalls.set(image, install);
|
|
754
|
+
}
|
|
755
|
+
await install;
|
|
756
|
+
return codexReviewerAppArmor.name;
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Times the steps of one create or start: `mark(step)` records the time since
|
|
760
|
+
* the previous mark, `line()` renders them for the runner log.
|
|
761
|
+
*/
|
|
762
|
+
export function createStepTimer(now = () => performance.now()) {
|
|
763
|
+
const startedAt = now();
|
|
764
|
+
let last = startedAt;
|
|
765
|
+
const steps = [];
|
|
766
|
+
return {
|
|
767
|
+
mark(step) {
|
|
768
|
+
const at = now();
|
|
769
|
+
steps.push([step, at - last]);
|
|
770
|
+
last = at;
|
|
771
|
+
},
|
|
772
|
+
line() {
|
|
773
|
+
const total = Math.round(last - startedAt);
|
|
774
|
+
return `${total} ms (${steps.map(([step, ms]) => `${step} ${Math.round(ms)}`).join(", ")})`;
|
|
775
|
+
},
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
function commandEnvelopeFromPayload(payload) {
|
|
779
|
+
return {
|
|
780
|
+
...(payload.commandGeneration !== undefined
|
|
781
|
+
? { commandGeneration: payload.commandGeneration }
|
|
782
|
+
: {}),
|
|
783
|
+
...(payload.operationId !== undefined ? { operationId: payload.operationId } : {}),
|
|
784
|
+
...(payload.desiredState !== undefined
|
|
785
|
+
? { desiredState: payload.desiredState }
|
|
786
|
+
: {}),
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
/**
|
|
790
|
+
* Attributes a container to the command generation that launched it. The
|
|
791
|
+
* launch record, not the immutable Docker labels, is authoritative: labels
|
|
792
|
+
* cannot be rewritten for a reused container, and a record that names a
|
|
793
|
+
* different container attributes nothing.
|
|
794
|
+
*/
|
|
795
|
+
function launchedGeneration(info, launched) {
|
|
796
|
+
if (!info?.Id || !launched)
|
|
797
|
+
return null;
|
|
798
|
+
return launched.containerId === info.Id ? launched.commandGeneration : null;
|
|
799
|
+
}
|
|
800
|
+
function launchedOperationId(info, launched) {
|
|
801
|
+
if (!info?.Id || !launched)
|
|
802
|
+
return null;
|
|
803
|
+
return launched.containerId === info.Id ? launched.operationId : null;
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Exits that prove the container never reached the workspace or the harness,
|
|
807
|
+
* so replacing or reusing it loses nothing. 75 is the fence refusing a
|
|
808
|
+
* superseded launch before touching the workspace; 76 is a later pre-harness
|
|
809
|
+
* failure; 77 is a Stop that claimed the boot before the harness was let go
|
|
810
|
+
* and had the Hub accept it; 79 is a Stop the entrypoint saved itself after
|
|
811
|
+
* stop-control could not, also accepted by the Hub. Without 75 here, the ordinary "stop soon after
|
|
812
|
+
* start" sequence would leave an environment permanently unable to start.
|
|
813
|
+
* 77 is here as well as in the Stop's own proof: the container reports the
|
|
814
|
+
* Stop to the Hub before Docker returns, so a restart can supersede the Stop
|
|
815
|
+
* command before it records anything.
|
|
816
|
+
*/
|
|
817
|
+
function safeReplacementProof(info, launched) {
|
|
818
|
+
const exitCode = info?.State?.ExitCode;
|
|
819
|
+
if (exitCode !== ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE
|
|
820
|
+
&& exitCode !== ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE
|
|
821
|
+
&& exitCode !== ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE
|
|
822
|
+
&& exitCode !== ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE) {
|
|
823
|
+
return null;
|
|
824
|
+
}
|
|
825
|
+
const commandGeneration = launchedGeneration(info, launched);
|
|
826
|
+
const operationId = launchedOperationId(info, launched);
|
|
827
|
+
return commandGeneration && operationId
|
|
828
|
+
? { commandGeneration, operationId, exitCode }
|
|
829
|
+
: null;
|
|
830
|
+
}
|
|
831
|
+
function failedBeforeHarnessProof(info, launched) {
|
|
832
|
+
if (info?.State?.ExitCode !== ENTRYPOINT_FAILED_BEFORE_HARNESS_EXIT_CODE)
|
|
833
|
+
return null;
|
|
834
|
+
const commandGeneration = launchedGeneration(info, launched);
|
|
835
|
+
const operationId = launchedOperationId(info, launched);
|
|
836
|
+
return commandGeneration && operationId ? { commandGeneration, operationId } : null;
|
|
837
|
+
}
|
|
838
|
+
/**
|
|
839
|
+
* The Stop that durably saved this exact stopped container, when nothing has
|
|
840
|
+
* started it since. It is the proof `createContainer` accepts to reuse or
|
|
841
|
+
* replace the container, so it holds however the commands after that Stop
|
|
842
|
+
* ended (for example a Start that never reached this runner, or one refused
|
|
843
|
+
* before it touched Docker).
|
|
844
|
+
*/
|
|
845
|
+
function durableStopProof(info, proof) {
|
|
846
|
+
if (!info?.Id || !proof?.containerId || proof.containerId !== info.Id)
|
|
847
|
+
return null;
|
|
848
|
+
if (!["stopped", "exited", "dead"].includes(getStatus(info)))
|
|
849
|
+
return null;
|
|
850
|
+
if (!startedBefore(info, proof.recordedAt))
|
|
851
|
+
return null;
|
|
852
|
+
return { stopOperationId: proof.stopOperationId, stopCommandGeneration: proof.stopCommandGeneration };
|
|
853
|
+
}
|
|
854
|
+
async function startContainer(slug) {
|
|
855
|
+
await runDocker(["start", containerName(slug)]);
|
|
856
|
+
}
|
|
857
|
+
function harnessSourceOf(info) {
|
|
858
|
+
return info.Mounts?.some((mount) => mount.Destination === HARNESS_DIST_CONTAINER_PATH) ? "overlay" : "image";
|
|
859
|
+
}
|
|
860
|
+
/** Docker reports "0001-01-01T00:00:00Z" for a time that has not happened. */
|
|
861
|
+
function dockerTimestamp(value) {
|
|
862
|
+
const parsed = Date.parse(value ?? "");
|
|
863
|
+
return Number.isFinite(parsed) && parsed > 0 ? value : undefined;
|
|
864
|
+
}
|
|
865
|
+
/** Create and start times from an inspect, as the startup trace reports them. */
|
|
866
|
+
function containerTimes(info) {
|
|
867
|
+
const createdAt = dockerTimestamp(info?.Created);
|
|
868
|
+
const startedAt = hasRun(info) ? dockerTimestamp(info?.State?.StartedAt) : undefined;
|
|
869
|
+
return {
|
|
870
|
+
...(createdAt ? { containerCreatedAt: createdAt } : {}),
|
|
871
|
+
...(startedAt ? { containerStartedAt: startedAt } : {}),
|
|
872
|
+
};
|
|
873
|
+
}
|
|
874
|
+
async function createContainer(payload, image, options) {
|
|
875
|
+
if (!payload.slug || !payload.repoUrl || !payload.envVars || typeof payload.envVars !== "object") {
|
|
876
|
+
throw new Error("slug, repoUrl, and envVars are required");
|
|
877
|
+
}
|
|
878
|
+
validateProjectEnvironmentDelivery(payload.envVars);
|
|
879
|
+
const mark = options.mark ?? (() => { });
|
|
880
|
+
const expectedGeneration = payload.commandGeneration ?? null;
|
|
881
|
+
const gitMirror = options.resolveGitMirror ? await options.resolveGitMirror(payload) : null;
|
|
882
|
+
mark("mirror");
|
|
883
|
+
const containerEnvVars = buildContainerEnvVars(payload, { gitMirror, terminalLane: options.terminalLane });
|
|
884
|
+
const envFileContents = buildHostLaunchFileContent(containerEnvVars);
|
|
885
|
+
// Resolved before the reuse decision because both the image comparison and
|
|
886
|
+
// the config hash are taken against the image this start would actually use.
|
|
887
|
+
const launchImage = requirePortableWorkspaceImage(requireProjectEnvironmentImage(image, payload.envVars), payload.envVars);
|
|
888
|
+
mark("image-check");
|
|
889
|
+
const envFile = createDockerEnvFile(containerEnvVars);
|
|
890
|
+
mark("env-file");
|
|
891
|
+
try {
|
|
892
|
+
options.guard();
|
|
893
|
+
const existing = await inspectContainer(payload.slug);
|
|
894
|
+
mark("inspect");
|
|
895
|
+
options.guard();
|
|
896
|
+
// Resolved before the hash so the `/workspace` argument is the same on
|
|
897
|
+
// every create or recreate of this slug.
|
|
898
|
+
const workspace = isIsolatedRun(payload.envVars) || !options.resolveWorkspaceVolume
|
|
899
|
+
? null
|
|
900
|
+
: await options.resolveWorkspaceVolume(payload, { containerExists: existing !== null });
|
|
901
|
+
mark("workspace-volume");
|
|
902
|
+
const harnessOverlayDir = options.resolveHarnessOverlay ? await options.resolveHarnessOverlay() : null;
|
|
903
|
+
const withWorkspaceKind = (outcome, info) => ({
|
|
904
|
+
...outcome,
|
|
905
|
+
...(workspace
|
|
906
|
+
? {
|
|
907
|
+
workspaceVolumeKind: workspace.kind,
|
|
908
|
+
...(workspace.stagedDependencies ? { stagedDependencies: workspace.stagedDependencies } : {}),
|
|
909
|
+
}
|
|
910
|
+
: {}),
|
|
911
|
+
// Read from the container, so a reused one reports what it really mounts.
|
|
912
|
+
...(info ? { harnessSource: harnessSourceOf(info) } : {}),
|
|
913
|
+
});
|
|
914
|
+
options.guard();
|
|
915
|
+
// The AppArmor profile is loaded into the kernel per create and does not
|
|
916
|
+
// survive a host reboot, so a reused sandboxed container needs it too.
|
|
917
|
+
const codexReviewerAppArmorProfile = usesCodexReviewerSandbox(containerEnvVars)
|
|
918
|
+
? await prepareReviewerAppArmor(launchImage, envFile.filePath)
|
|
919
|
+
: undefined;
|
|
920
|
+
options.guard();
|
|
921
|
+
const runArgsOptions = {
|
|
922
|
+
...(codexReviewerAppArmorProfile ? { codexReviewerAppArmorProfile } : {}),
|
|
923
|
+
fenceDirectory: options.fenceDirectory,
|
|
924
|
+
...(gitMirror ? { gitMirror } : {}),
|
|
925
|
+
...(workspace ? { workspaceVolume: workspace.volume } : {}),
|
|
926
|
+
...(harnessOverlayDir ? { harnessOverlayDir } : {}),
|
|
927
|
+
};
|
|
928
|
+
const configHash = computeContainerConfigHash(payload, launchImage, envFile.filePath, runArgsOptions);
|
|
929
|
+
mark("config-hash");
|
|
930
|
+
// Publishing the launch and starting are one step: the record names the
|
|
931
|
+
// container before any process in it runs, and it is written under the
|
|
932
|
+
// per-slug queue after asserting the fence, so a command accepted on the
|
|
933
|
+
// HTTP path cannot claim a container it did not start.
|
|
934
|
+
const launchAndStart = async (containerId) => {
|
|
935
|
+
options.guard();
|
|
936
|
+
options.recordLaunch(containerId, envFileContents);
|
|
937
|
+
mark("record-launch");
|
|
938
|
+
options.guard();
|
|
939
|
+
await startContainer(payload.slug);
|
|
940
|
+
mark("docker-start");
|
|
941
|
+
};
|
|
942
|
+
const createFresh = async () => {
|
|
943
|
+
options.guard();
|
|
944
|
+
const args = buildContainerRunArgs(payload, launchImage, envFile.filePath, {
|
|
945
|
+
...runArgsOptions,
|
|
946
|
+
configHash,
|
|
947
|
+
});
|
|
948
|
+
const containerId = (await runDocker(args)).trim();
|
|
949
|
+
const createdAt = new Date().toISOString();
|
|
950
|
+
mark("docker-create");
|
|
951
|
+
if (!/^[0-9a-f]{64}$/.test(containerId)) {
|
|
952
|
+
throw new Error("Docker did not report a container id for the created runner.");
|
|
953
|
+
}
|
|
954
|
+
await launchAndStart(containerId);
|
|
955
|
+
const startedAt = new Date().toISOString();
|
|
956
|
+
// If a newer Stop won while Docker was starting, leave the live container
|
|
957
|
+
// in place. The per-slug command queue will run that fenced Stop next so
|
|
958
|
+
// it can complete the normal durable workspace-save flow.
|
|
959
|
+
options.guard();
|
|
960
|
+
// No inspect after the start (it took up to 1.4 s live): everything the
|
|
961
|
+
// answer carries is already known for a container this create made.
|
|
962
|
+
// `docker start` succeeded, so it was started; a container that exits at
|
|
963
|
+
// once is reported by its entrypoint and the Hub's start window, as
|
|
964
|
+
// before (nothing read the inspected status). The trace times are the
|
|
965
|
+
// runner's, taken as `docker create` and `docker start` returned.
|
|
966
|
+
return {
|
|
967
|
+
runnerId: containerName(payload.slug),
|
|
968
|
+
status: "running",
|
|
969
|
+
containerReuse: "created",
|
|
970
|
+
containerCreatedAt: createdAt,
|
|
971
|
+
containerStartedAt: startedAt,
|
|
972
|
+
...(workspace
|
|
973
|
+
? {
|
|
974
|
+
workspaceVolumeKind: workspace.kind,
|
|
975
|
+
...(workspace.stagedDependencies ? { stagedDependencies: workspace.stagedDependencies } : {}),
|
|
976
|
+
}
|
|
977
|
+
: {}),
|
|
978
|
+
harnessSource: harnessOverlayDir ? "overlay" : "image",
|
|
979
|
+
};
|
|
980
|
+
};
|
|
981
|
+
if (existing) {
|
|
982
|
+
const launched = options.readLaunched();
|
|
983
|
+
const existingGeneration = launchedGeneration(existing, launched);
|
|
984
|
+
const status = getStatus(existing);
|
|
985
|
+
if (status === "created") {
|
|
986
|
+
// A runner crash between `docker create` and `docker start`. An exact
|
|
987
|
+
// retry of the command that created it starts it; anything else removes
|
|
988
|
+
// it, because nothing ever ran.
|
|
989
|
+
const matchesThisCommand = Boolean(expectedGeneration !== null
|
|
990
|
+
&& launched
|
|
991
|
+
&& existing.Id
|
|
992
|
+
&& launched.containerId === existing.Id
|
|
993
|
+
&& launched.commandGeneration === expectedGeneration
|
|
994
|
+
&& launched.operationId === payload.operationId);
|
|
995
|
+
if (matchesThisCommand) {
|
|
996
|
+
options.guard();
|
|
997
|
+
await startContainer(payload.slug);
|
|
998
|
+
options.guard();
|
|
999
|
+
const info = await inspectContainer(payload.slug);
|
|
1000
|
+
return withWorkspaceKind({
|
|
1001
|
+
runnerId: containerName(payload.slug),
|
|
1002
|
+
status: getStatus(info),
|
|
1003
|
+
containerReuse: "reused",
|
|
1004
|
+
...containerTimes(info),
|
|
1005
|
+
}, info);
|
|
1006
|
+
}
|
|
1007
|
+
await removeStoppedContainer(payload.slug);
|
|
1008
|
+
options.guard();
|
|
1009
|
+
return await createFresh();
|
|
1010
|
+
}
|
|
1011
|
+
if (status === "running") {
|
|
1012
|
+
if (expectedGeneration !== null && existingGeneration === expectedGeneration) {
|
|
1013
|
+
requireProjectEnvironmentContainer(existing, payload.envVars);
|
|
1014
|
+
requirePortableWorkspaceContainer(existing, payload.envVars);
|
|
1015
|
+
return withWorkspaceKind({
|
|
1016
|
+
runnerId: containerName(payload.slug),
|
|
1017
|
+
status: "running",
|
|
1018
|
+
containerReuse: "reused",
|
|
1019
|
+
...containerTimes(existing),
|
|
1020
|
+
}, existing);
|
|
1021
|
+
}
|
|
1022
|
+
throw new RunnerCommandFenceError(`A live runner already exists for ${payload.slug}; refusing to replace it.`, "runner_command_conflict");
|
|
1023
|
+
}
|
|
1024
|
+
const persistenceProof = options.readWorkspaceSyncedStop();
|
|
1025
|
+
const replacement = safeReplacementProof(existing, launched);
|
|
1026
|
+
const acknowledgedPersistence = Boolean(expectedGeneration !== null
|
|
1027
|
+
&& persistenceProof
|
|
1028
|
+
&& persistenceProof.stopCommandGeneration < expectedGeneration
|
|
1029
|
+
&& (
|
|
1030
|
+
// A durable stop of this container is proof for this container,
|
|
1031
|
+
// whatever generation the launch record now names.
|
|
1032
|
+
(persistenceProof.containerId !== undefined
|
|
1033
|
+
&& existing.Id !== undefined
|
|
1034
|
+
&& persistenceProof.containerId === existing.Id
|
|
1035
|
+
&& startedBefore(existing, persistenceProof.recordedAt))
|
|
1036
|
+
|| (existingGeneration !== null
|
|
1037
|
+
&& persistenceProof.runnerCommandGeneration === existingGeneration)));
|
|
1038
|
+
const safePreHarnessFailure = Boolean(expectedGeneration !== null
|
|
1039
|
+
&& replacement
|
|
1040
|
+
&& replacement.commandGeneration < expectedGeneration);
|
|
1041
|
+
if (!acknowledgedPersistence && !safePreHarnessFailure) {
|
|
1042
|
+
throw new RunnerCommandFenceError(`The stopped runner for ${payload.slug} has no matching workspace persistence acknowledgement.`, "runner_command_conflict");
|
|
1043
|
+
}
|
|
1044
|
+
options.guard();
|
|
1045
|
+
const pinnedImageId = await resolveImageId(launchImage);
|
|
1046
|
+
options.guard();
|
|
1047
|
+
let reusable = Boolean(
|
|
1048
|
+
// A previous recovery may already have materialized files, even when
|
|
1049
|
+
// native restoration failed. A new recovery always starts empty.
|
|
1050
|
+
payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined
|
|
1051
|
+
&& existing.Id
|
|
1052
|
+
&& expectedGeneration !== null
|
|
1053
|
+
&& existingGeneration !== null
|
|
1054
|
+
&& existingGeneration < expectedGeneration
|
|
1055
|
+
&& pinnedImageId !== null
|
|
1056
|
+
&& existing.Image === pinnedImageId
|
|
1057
|
+
&& containerConfigHash(existing) === configHash
|
|
1058
|
+
&& (hasRun(existing) || replacement !== null));
|
|
1059
|
+
if (reusable) {
|
|
1060
|
+
try {
|
|
1061
|
+
// A reused container keeps the environment it was created with, so a
|
|
1062
|
+
// project-environment or saved-workspace selection that no longer
|
|
1063
|
+
// matches has to be answered with a fresh container rather than a
|
|
1064
|
+
// failed start.
|
|
1065
|
+
requireProjectEnvironmentContainer(existing, payload.envVars);
|
|
1066
|
+
requirePortableWorkspaceContainer(existing, payload.envVars);
|
|
1067
|
+
}
|
|
1068
|
+
catch {
|
|
1069
|
+
reusable = false;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
if (reusable) {
|
|
1073
|
+
try {
|
|
1074
|
+
await launchAndStart(existing.Id);
|
|
1075
|
+
options.guard();
|
|
1076
|
+
const info = await inspectContainer(payload.slug);
|
|
1077
|
+
return withWorkspaceKind({
|
|
1078
|
+
runnerId: containerName(payload.slug),
|
|
1079
|
+
status: getStatus(info),
|
|
1080
|
+
containerReuse: "reused",
|
|
1081
|
+
...containerTimes(info),
|
|
1082
|
+
}, info);
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
if (error instanceof RunnerCommandFenceError)
|
|
1086
|
+
throw error;
|
|
1087
|
+
// Eligibility was verified a moment ago, so a failed start is this
|
|
1088
|
+
// container's problem. Replace it inside the same execution, while
|
|
1089
|
+
// the proof that made it replaceable still holds.
|
|
1090
|
+
console.error(`[runner] reuse of the stopped runner for ${payload.slug} failed; recreating:`, redactEnvValues(error instanceof Error ? error.message : String(error), containerEnvVars));
|
|
1091
|
+
await forceRemoveContainer(payload.slug);
|
|
1092
|
+
options.guard();
|
|
1093
|
+
return await createFresh();
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
await removeStoppedContainer(payload.slug);
|
|
1097
|
+
options.guard();
|
|
1098
|
+
}
|
|
1099
|
+
return await createFresh();
|
|
1100
|
+
}
|
|
1101
|
+
catch (error) {
|
|
1102
|
+
if (error instanceof RunnerCommandFenceError)
|
|
1103
|
+
throw error;
|
|
1104
|
+
throw redactError(error, containerEnvVars);
|
|
1105
|
+
}
|
|
1106
|
+
finally {
|
|
1107
|
+
envFile.cleanup();
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
async function inspectAppliedRunningCommand(record, fenceStore, envVars) {
|
|
1111
|
+
fenceStore.assertCurrent(record);
|
|
1112
|
+
const info = await inspectContainer(record.slug);
|
|
1113
|
+
fenceStore.assertCurrent(record);
|
|
1114
|
+
const launched = fenceStore.readLaunched(record.slug);
|
|
1115
|
+
if (!info || launchedGeneration(info, launched) !== record.commandGeneration) {
|
|
1116
|
+
throw new RunnerCommandFenceError(`Runner command generation ${record.commandGeneration} was applied, but its exact runner is unavailable.`, "runner_command_conflict");
|
|
1117
|
+
}
|
|
1118
|
+
requireProjectEnvironmentContainer(info, envVars);
|
|
1119
|
+
requirePortableWorkspaceContainer(info, envVars);
|
|
1120
|
+
return {
|
|
1121
|
+
runnerId: containerName(record.slug),
|
|
1122
|
+
status: getStatus(info),
|
|
1123
|
+
containerReuse: "reused",
|
|
1124
|
+
...containerTimes(info),
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
async function ensureStarted(slug, payload, image, options) {
|
|
1128
|
+
if (payload?.repoUrl && payload?.envVars && typeof payload.envVars === "object") {
|
|
1129
|
+
return createContainer({
|
|
1130
|
+
slug,
|
|
1131
|
+
repoUrl: payload.repoUrl,
|
|
1132
|
+
envVars: payload.envVars,
|
|
1133
|
+
...(payload.commandGeneration ? { commandGeneration: payload.commandGeneration } : {}),
|
|
1134
|
+
...(payload.operationId ? { operationId: payload.operationId } : {}),
|
|
1135
|
+
...(payload.desiredState ? { desiredState: payload.desiredState } : {}),
|
|
1136
|
+
}, image, options);
|
|
1137
|
+
}
|
|
1138
|
+
options.guard();
|
|
1139
|
+
const existing = await inspectContainer(slug);
|
|
1140
|
+
options.guard();
|
|
1141
|
+
if (!existing) {
|
|
1142
|
+
throw new Error(`Container ${containerName(slug)} does not exist`);
|
|
1143
|
+
}
|
|
1144
|
+
requireProjectEnvironmentContainer(existing, payload?.envVars);
|
|
1145
|
+
requirePortableWorkspaceContainer(existing, payload?.envVars);
|
|
1146
|
+
options.guard();
|
|
1147
|
+
return {
|
|
1148
|
+
runnerId: containerName(slug),
|
|
1149
|
+
status: getStatus(existing),
|
|
1150
|
+
containerReuse: "reused",
|
|
1151
|
+
...containerTimes(existing),
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
function quoteShellString(value) {
|
|
1155
|
+
return `'${value.replace(/'/g, `'\"'\"'`)}'`;
|
|
1156
|
+
}
|
|
1157
|
+
function buildStopClientScript(args) {
|
|
1158
|
+
return (`set -eu; ` +
|
|
1159
|
+
`unset NODE_OPTIONS NODE_PATH GLIBC_TUNABLES GCONV_PATH LOCPATH NLSPATH; ` +
|
|
1160
|
+
`for name in "\${!LD_@}"; do unset "$name"; done; ` +
|
|
1161
|
+
`exec /usr/local/bin/tiller-controller-node /stop-control-client.mjs ${args.map((arg) => arg.startsWith("--") ? arg : quoteShellString(arg)).join(" ")}`);
|
|
1162
|
+
}
|
|
1163
|
+
export function buildPrepareDurableStopScript(stopOpId, options = {}) {
|
|
1164
|
+
const trimmedStopOpId = stopOpId?.trim();
|
|
1165
|
+
if (!trimmedStopOpId)
|
|
1166
|
+
throw new Error("A lifecycle Stop operation ID is required");
|
|
1167
|
+
const idleClaimId = options.idleClaimId?.trim();
|
|
1168
|
+
return buildStopClientScript([
|
|
1169
|
+
...(options.fenceOnly ? ["--fence"] : []),
|
|
1170
|
+
...(idleClaimId ? ["--idle-claim", idleClaimId] : []),
|
|
1171
|
+
trimmedStopOpId,
|
|
1172
|
+
]);
|
|
1173
|
+
}
|
|
1174
|
+
/** stop-control-client: the idle Stop's claim is no longer held; nothing was fenced. */
|
|
1175
|
+
export const STOP_CLIENT_IDLE_CLAIM_SUPERSEDED_EXIT_CODE = 13;
|
|
1176
|
+
/**
|
|
1177
|
+
* How long a host idle claim may fence the harness's input before a Stop
|
|
1178
|
+
* turns it into a fence. The Hub asks for the Stop right after the claim; a
|
|
1179
|
+
* Hub that never does must not leave the terminal refusing keystrokes.
|
|
1180
|
+
*/
|
|
1181
|
+
export const HOST_IDLE_CLAIM_TTL_MS = 120_000;
|
|
1182
|
+
function normalizeIdleStopPreparation(value, idleTimeoutMs) {
|
|
1183
|
+
const record = value && typeof value === "object" ? value : {};
|
|
1184
|
+
const claimId = typeof record.claimId === "string" && record.claimId.trim() ? record.claimId.trim() : undefined;
|
|
1185
|
+
const eligible = record.eligible === true && Boolean(claimId);
|
|
1186
|
+
const remaining = typeof record.remainingIdleMs === "number" && Number.isFinite(record.remainingIdleMs)
|
|
1187
|
+
? Math.max(0, Math.ceil(record.remainingIdleMs))
|
|
1188
|
+
: idleTimeoutMs;
|
|
1189
|
+
return {
|
|
1190
|
+
eligible,
|
|
1191
|
+
reason: typeof record.reason === "string" && record.reason ? record.reason : eligible ? "eligible" : "not_eligible",
|
|
1192
|
+
remainingIdleMs: eligible ? 0 : remaining,
|
|
1193
|
+
...(eligible ? { claimId } : {}),
|
|
1194
|
+
...(typeof record.idleSince === "number" && Number.isFinite(record.idleSince) ? { idleSince: record.idleSince } : {}),
|
|
1195
|
+
...(typeof record.error === "string" && record.error ? { error: record.error.slice(0, 500) } : {}),
|
|
1196
|
+
};
|
|
1197
|
+
}
|
|
1198
|
+
/**
|
|
1199
|
+
* Host idle stop, phase one: the container's stop-control claims the idle
|
|
1200
|
+
* stop (terminal output counts as activity) and saves the workspace. It
|
|
1201
|
+
* changes nothing else; a claim that is not turned into a Stop expires.
|
|
1202
|
+
*/
|
|
1203
|
+
export async function prepareIdleStopInContainer(slug, idleTimeoutMs, claimTtlMs = HOST_IDLE_CLAIM_TTL_MS) {
|
|
1204
|
+
const output = await runDocker([
|
|
1205
|
+
"exec", containerName(slug), "bash", "-c",
|
|
1206
|
+
buildStopClientScript(["--idle-prepare", String(Math.floor(idleTimeoutMs)), String(Math.floor(claimTtlMs))]),
|
|
1207
|
+
]);
|
|
1208
|
+
const line = output.trim().split("\n").filter(Boolean).at(-1) ?? "";
|
|
1209
|
+
let parsed;
|
|
1210
|
+
try {
|
|
1211
|
+
parsed = JSON.parse(line);
|
|
1212
|
+
}
|
|
1213
|
+
catch {
|
|
1214
|
+
throw new Error("The container's idle stop preparation gave no answer");
|
|
1215
|
+
}
|
|
1216
|
+
return normalizeIdleStopPreparation(parsed, idleTimeoutMs);
|
|
1217
|
+
}
|
|
1218
|
+
/** Releases an idle claim the Hub will not use. Best effort: the claim also expires. */
|
|
1219
|
+
export async function releaseIdleStopInContainer(slug, claimId) {
|
|
1220
|
+
await runDocker(["exec", containerName(slug), "bash", "-c", buildStopClientScript(["--idle-release", claimId])]);
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* The idle Stop's decision, made before the runner accepts the Stop: the
|
|
1224
|
+
* harness fences only while it still holds the claim and is still idle.
|
|
1225
|
+
* Anything else (the session became active, the claim expired, the
|
|
1226
|
+
* container cannot be reached) refuses the Stop with nothing changed.
|
|
1227
|
+
*/
|
|
1228
|
+
export async function fenceIdleStopInContainer(slug, stopOpId, idleClaimId) {
|
|
1229
|
+
try {
|
|
1230
|
+
await runDocker(["exec", containerName(slug), "bash", "-c", buildPrepareDurableStopScript(stopOpId, { fenceOnly: true, idleClaimId })]);
|
|
1231
|
+
return "fenced";
|
|
1232
|
+
}
|
|
1233
|
+
catch {
|
|
1234
|
+
return "superseded";
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Fences the container for this Stop before the runner acknowledges it: the
|
|
1239
|
+
* boot is claimed for the Stop, or a harness already let go is told to spawn
|
|
1240
|
+
* and report nothing more. Never throws; a container that cannot be fenced
|
|
1241
|
+
* is handled by the preparation that follows.
|
|
1242
|
+
*/
|
|
1243
|
+
export async function fenceStopInContainer(slug, stopOpId, options = {}) {
|
|
1244
|
+
const script = buildPrepareDurableStopScript(stopOpId, { fenceOnly: true });
|
|
1245
|
+
const deadline = Date.now() + (options.bootSetupWaitMs ?? STOP_BOOT_SETUP_WAIT_MS);
|
|
1246
|
+
for (;;) {
|
|
1247
|
+
try {
|
|
1248
|
+
await runDocker(["exec", containerName(slug), "bash", "-c", script]);
|
|
1249
|
+
return "fenced";
|
|
1250
|
+
}
|
|
1251
|
+
catch (error) {
|
|
1252
|
+
const code = error?.code;
|
|
1253
|
+
if (code === String(STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE) && Date.now() < deadline) {
|
|
1254
|
+
await new Promise((resolve) => setTimeout(resolve, options.retryMs ?? 100));
|
|
1255
|
+
continue;
|
|
1256
|
+
}
|
|
1257
|
+
return "unfenced";
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Exit codes of /stop-control-client.mjs. Kept in step with that file.
|
|
1263
|
+
* 10: the Stop claimed the container's startup before the harness was let go,
|
|
1264
|
+
* so no agent ever ran there; the entrypoint confirms the save to the Hub
|
|
1265
|
+
* (the Hub already holds the whole workspace) and exits when stopped.
|
|
1266
|
+
* 11: the harness was let go, but stop-control could not be reached or said
|
|
1267
|
+
* the workspace is not hydrated.
|
|
1268
|
+
*/
|
|
1269
|
+
export const STOP_CLIENT_STOPPED_BEFORE_RELEASE_EXIT_CODE = 10;
|
|
1270
|
+
export const STOP_CLIENT_STOP_CONTROL_UNAVAILABLE_EXIT_CODE = 11;
|
|
1271
|
+
/** The container's entrypoint has not set up its boot yet; ask again shortly. */
|
|
1272
|
+
export const STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE = 12;
|
|
1273
|
+
/** How long a Stop waits for a just-started container to set up its boot. */
|
|
1274
|
+
export const STOP_BOOT_SETUP_WAIT_MS = 5_000;
|
|
1275
|
+
/**
|
|
1276
|
+
* Exit code of a container whose entrypoint ended a Stop that arrived before
|
|
1277
|
+
* its harness was let go, after the Hub accepted the save confirmation.
|
|
1278
|
+
*/
|
|
1279
|
+
export const ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE = 77;
|
|
1280
|
+
/**
|
|
1281
|
+
* Exit code of a container whose entrypoint saved the workspace itself for a
|
|
1282
|
+
* Stop that stop-control never prepared, after the Hub acknowledged the save.
|
|
1283
|
+
*/
|
|
1284
|
+
export const ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE = 79;
|
|
1285
|
+
/** Time a container stopped outright gets to confirm the save and report before Docker kills it. */
|
|
1286
|
+
export const OUTRIGHT_STOP_GRACE_SECONDS = 30;
|
|
1287
|
+
/**
|
|
1288
|
+
* Time a prepared container gets to end its harness (bounded at 15 s) and
|
|
1289
|
+
* stop-control (10 s) and report the Stop to the Hub. Docker's implicit 10 s
|
|
1290
|
+
* could kill it before the report, leaving the Hub waiting for its alarm.
|
|
1291
|
+
*/
|
|
1292
|
+
export const PREPARED_STOP_GRACE_SECONDS = 30;
|
|
1293
|
+
/** A Scribe's container holds no workspace save; Docker's usual grace. */
|
|
1294
|
+
export const SCRIBE_STOP_GRACE_SECONDS = 10;
|
|
1295
|
+
/**
|
|
1296
|
+
* Longest an inspection waits for the slug's in-flight Stop before answering
|
|
1297
|
+
* (see `waitForInFlightStop`). It covers the seconds between the container
|
|
1298
|
+
* reporting its Stop to the Hub and `docker stop` returning here, and stays
|
|
1299
|
+
* well inside the Hub's 15 s runner-control wait.
|
|
1300
|
+
*/
|
|
1301
|
+
export const INSPECTION_STOP_WAIT_MS = 5_000;
|
|
1302
|
+
function dockerExecUnreachable(error) {
|
|
1303
|
+
const code = error?.code;
|
|
1304
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1305
|
+
return code === "125" || code === "126" || code === "127"
|
|
1306
|
+
|| /is not running|no such container|is restarting|is paused|cannot exec|OCI runtime exec failed/i.test(message);
|
|
1307
|
+
}
|
|
1308
|
+
export async function prepareDurableStop(slug, stopOpId, options = {}) {
|
|
1309
|
+
// An idle Stop names its claim so stop-control can reuse the idle save. The
|
|
1310
|
+
// decision was already made at the fence, so a claim stop-control no longer
|
|
1311
|
+
// knows (it restarted) only costs a fresh save, never the Stop.
|
|
1312
|
+
let script = buildPrepareDurableStopScript(stopOpId, { idleClaimId: options.idleClaimId });
|
|
1313
|
+
const deadline = Date.now() + (options.bootSetupWaitMs ?? STOP_BOOT_SETUP_WAIT_MS);
|
|
1314
|
+
for (;;) {
|
|
1315
|
+
try {
|
|
1316
|
+
await runDocker(["exec", containerName(slug), "bash", "-c", script]);
|
|
1317
|
+
return { outcome: "prepared" };
|
|
1318
|
+
}
|
|
1319
|
+
catch (error) {
|
|
1320
|
+
const code = error?.code;
|
|
1321
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1322
|
+
if (code === String(STOP_CLIENT_IDLE_CLAIM_SUPERSEDED_EXIT_CODE) && options.idleClaimId) {
|
|
1323
|
+
options = { ...options, idleClaimId: null };
|
|
1324
|
+
script = buildPrepareDurableStopScript(stopOpId);
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
if (code === String(STOP_CLIENT_BOOT_NOT_SET_UP_EXIT_CODE)) {
|
|
1328
|
+
// The container started a moment ago; its entrypoint sets up the
|
|
1329
|
+
// boot within a few hundred ms, and then the Stop can claim it.
|
|
1330
|
+
if (Date.now() < deadline) {
|
|
1331
|
+
await new Promise((resolve) => setTimeout(resolve, options.retryMs ?? 100));
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
return { outcome: "stopped-before-release", detail };
|
|
1335
|
+
}
|
|
1336
|
+
if (code === String(STOP_CLIENT_STOPPED_BEFORE_RELEASE_EXIT_CODE))
|
|
1337
|
+
return { outcome: "stopped-before-release", detail };
|
|
1338
|
+
if (code === String(STOP_CLIENT_STOP_CONTROL_UNAVAILABLE_EXIT_CODE) || dockerExecUnreachable(error))
|
|
1339
|
+
return { outcome: "unavailable", detail };
|
|
1340
|
+
// Any other failure, Docker's included: a Stop that was accepted still
|
|
1341
|
+
// ends with the container stopped.
|
|
1342
|
+
return { outcome: "failed", detail };
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
/** Long enough for the environment's own boot to write its base into the mirror. */
|
|
1347
|
+
export const STAGE_AFTER_CREATE_DELAY_MS = 60 * 1000;
|
|
1348
|
+
/**
|
|
1349
|
+
* Long enough for an adopting environment to reach ready (about 3 s after its
|
|
1350
|
+
* container starts) before its replacement's stage run and dependency install
|
|
1351
|
+
* compete with it for CPU and disk. The runner has no ready signal of its own.
|
|
1352
|
+
*/
|
|
1353
|
+
export const REFILL_AFTER_ADOPTION_DELAY_MS = 10 * 1000;
|
|
1354
|
+
const RUNNER_VOLUME_RECONCILE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
1355
|
+
export async function startRunnerServer(config) {
|
|
1356
|
+
const { port, image } = config;
|
|
1357
|
+
const identity = Object.freeze({ instanceId: randomUUID(), packageVersion: HOST_PACKAGE_VERSION, runtimeImage: image, runtimePlatform: readRuntimeImagePlatform(image) });
|
|
1358
|
+
const fenceStore = new RunnerCommandFenceStore(config.localStateDir);
|
|
1359
|
+
const localStateDir = config.localStateDir ?? LOCAL_STATE_DIR;
|
|
1360
|
+
const apiToken = config.apiToken || createRunnerApiToken();
|
|
1361
|
+
// One record of every volume the runner manages.
|
|
1362
|
+
const inventory = new RunnerInventory(localStateDir);
|
|
1363
|
+
try {
|
|
1364
|
+
const migrated = inventory.migrate(fenceStore.listWorkspaceVolumeRecords());
|
|
1365
|
+
if (migrated.length)
|
|
1366
|
+
console.log(`[runner] moved ${migrated.map((path) => basename(path)).join(" and ")} into the runner inventory`);
|
|
1367
|
+
}
|
|
1368
|
+
catch (error) {
|
|
1369
|
+
console.error("[runner] could not migrate runner state into the inventory:", error instanceof Error ? error.message : String(error));
|
|
1370
|
+
}
|
|
1371
|
+
const budgetBytes = config.stagedWorkspaceBudgetBytes
|
|
1372
|
+
?? parseByteSize(process.env[envVar("STAGED_WORKSPACE_BUDGET_BYTES")])
|
|
1373
|
+
?? STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES;
|
|
1374
|
+
// Staging holds disk ahead of use, so it runs only where there is room:
|
|
1375
|
+
// decided once here from free space on the Docker data volume, and checked
|
|
1376
|
+
// again before each stage run. A budget of 0 (no eviction) is measured
|
|
1377
|
+
// against the default budget.
|
|
1378
|
+
const diskProbe = config.dockerDiskProbe ?? createDockerDiskProbe(image, runDocker);
|
|
1379
|
+
const stagingDecision = decideStagingAtStartup(await diskProbe().catch(() => null), budgetBytes || STAGED_WORKSPACE_DEFAULT_BUDGET_BYTES);
|
|
1380
|
+
const runnerCapabilities = Object.freeze(resolveRunnerCapabilities(image, {
|
|
1381
|
+
stagedWorkspaces: stagingDecision.enabled,
|
|
1382
|
+
terminalLane: Boolean(config.terminalLane),
|
|
1383
|
+
}));
|
|
1384
|
+
// The build-and-test job runs in the runner's own image, given nothing but an app's files.
|
|
1385
|
+
const gadgetJobs = new GadgetJobs({
|
|
1386
|
+
image,
|
|
1387
|
+
localStateDir,
|
|
1388
|
+
runDocker,
|
|
1389
|
+
...(config.gadgetJob?.runContainer ? { runContainer: config.gadgetJob.runContainer } : {}),
|
|
1390
|
+
...(config.gadgetJob?.timeoutMs ? { timeoutMs: config.gadgetJob.timeoutMs } : {}),
|
|
1391
|
+
log: (message) => console.log(message),
|
|
1392
|
+
});
|
|
1393
|
+
const terminalLaneTickets = new TerminalLaneTickets();
|
|
1394
|
+
const terminalLaneRelays = new TerminalLaneRelays();
|
|
1395
|
+
/** Stop and Delete end every lane of the environment and void its unused tickets. */
|
|
1396
|
+
const closeTerminalLanes = (slug) => {
|
|
1397
|
+
terminalLaneTickets.revoke(slug);
|
|
1398
|
+
terminalLaneRelays.closeAll(slug);
|
|
1399
|
+
};
|
|
1400
|
+
const stagedWorkspaces = stagingDecision.enabled
|
|
1401
|
+
? new StagedWorkspaces({
|
|
1402
|
+
inventory,
|
|
1403
|
+
image,
|
|
1404
|
+
runDocker,
|
|
1405
|
+
canonicalRepositoryUrl: canonicalGitHubRepositoryUrl,
|
|
1406
|
+
mirrorVolumeName,
|
|
1407
|
+
dependencies: { cacheMounts: PACKAGE_CACHE_MOUNTS, maxBytes: budgetBytes },
|
|
1408
|
+
diskRefusal: async () => stageRunDiskRefusal(await diskProbe().catch(() => null)),
|
|
1409
|
+
})
|
|
1410
|
+
: null;
|
|
1411
|
+
/**
|
|
1412
|
+
* Pending delayed stages by `repo@base`, with the environments waiting on
|
|
1413
|
+
* each. `null` stands for one no Stop or Delete may cancel.
|
|
1414
|
+
*/
|
|
1415
|
+
const stageTimers = new Map();
|
|
1416
|
+
const refillTimers = new Set();
|
|
1417
|
+
// Development-only options, honoured only while the Hub says it is a
|
|
1418
|
+
// development build (`development-runner.ts`).
|
|
1419
|
+
const development = { ...readDevelopmentRunnerOptions(), ...config.development };
|
|
1420
|
+
const developmentHub = new DevelopmentHubGate(config.hubBuildChannel ?? (() => fetchHubBuildChannel(HUB_URL, hubControlHeaders)));
|
|
1421
|
+
const developmentKnobs = development.stageDelayMs !== null || development.refillDelayMs !== null;
|
|
1422
|
+
const stagingHold = new StagingHold();
|
|
1423
|
+
if (developmentKnobs || development.harnessDir) {
|
|
1424
|
+
void developmentHub.isDevelopment().then((isDevelopment) => {
|
|
1425
|
+
const names = [
|
|
1426
|
+
development.harnessDir ? `${envVar("DEV_HARNESS_DIR")}=${development.harnessDir}` : null,
|
|
1427
|
+
development.stageDelayMs !== null ? `${envVar("STAGE_DELAY_MS")}=${development.stageDelayMs}` : null,
|
|
1428
|
+
development.refillDelayMs !== null ? `${envVar("REFILL_DELAY_MS")}=${development.refillDelayMs}` : null,
|
|
1429
|
+
].filter(Boolean).join(", ");
|
|
1430
|
+
console.log(isDevelopment
|
|
1431
|
+
? `[runner] development options on (the Hub is a development build): ${names}`
|
|
1432
|
+
: `[runner] development options ignored, the Hub is not a known development build: ${names}`);
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
/** A development delay when one is set and the Hub was last seen as a development build. */
|
|
1436
|
+
const developmentDelay = (value, fallback) => value !== null && developmentHub.knownDevelopment() ? value : fallback;
|
|
1437
|
+
async function resolveHarnessOverlay() {
|
|
1438
|
+
if (!development.harnessDir)
|
|
1439
|
+
return null;
|
|
1440
|
+
if (!(await developmentHub.isDevelopment()))
|
|
1441
|
+
throw new Error(HARNESS_OVERLAY_REFUSED);
|
|
1442
|
+
return validateHarnessOverlayDir(development.harnessDir);
|
|
1443
|
+
}
|
|
1444
|
+
const preparations = new Map();
|
|
1445
|
+
const gitMirrorTimers = [];
|
|
1446
|
+
/** Bookkeeping never fails a start; a missed update is repaired by the next reconciliation. */
|
|
1447
|
+
const updateInventory = (change) => {
|
|
1448
|
+
try {
|
|
1449
|
+
change(inventory);
|
|
1450
|
+
}
|
|
1451
|
+
catch (error) {
|
|
1452
|
+
console.error("[runner] could not update the runner inventory:", error instanceof Error ? error.message : String(error));
|
|
1453
|
+
}
|
|
1454
|
+
};
|
|
1455
|
+
const commandTails = new Map();
|
|
1456
|
+
const commandExecutions = new Map();
|
|
1457
|
+
const stopAcknowledgements = new Map();
|
|
1458
|
+
/** The Stop each slug is still executing, settled either way. */
|
|
1459
|
+
const inFlightStops = new Map();
|
|
1460
|
+
/** The idle claim an accepted idle Stop decided on, for its prepare (save reuse). */
|
|
1461
|
+
const idleStopClaims = new Map();
|
|
1462
|
+
function commandKey(record) {
|
|
1463
|
+
return `${record.slug}:${record.commandGeneration}:${record.operationId}:${record.desiredState}`;
|
|
1464
|
+
}
|
|
1465
|
+
function commandResponse(record) {
|
|
1466
|
+
return {
|
|
1467
|
+
commandGeneration: record.commandGeneration,
|
|
1468
|
+
operationId: record.operationId,
|
|
1469
|
+
desiredState: record.desiredState,
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
function scheduleCommand(record, effect) {
|
|
1473
|
+
const key = commandKey(record);
|
|
1474
|
+
const existing = commandExecutions.get(key);
|
|
1475
|
+
if (existing)
|
|
1476
|
+
return existing;
|
|
1477
|
+
const previous = commandTails.get(record.slug) ?? Promise.resolve();
|
|
1478
|
+
const execution = previous.catch(() => { }).then(effect);
|
|
1479
|
+
const tail = execution.then(() => undefined, () => undefined);
|
|
1480
|
+
commandTails.set(record.slug, tail);
|
|
1481
|
+
commandExecutions.set(key, execution);
|
|
1482
|
+
void execution.then(() => {
|
|
1483
|
+
if (commandExecutions.get(key) === execution)
|
|
1484
|
+
commandExecutions.delete(key);
|
|
1485
|
+
}, () => {
|
|
1486
|
+
if (commandExecutions.get(key) === execution)
|
|
1487
|
+
commandExecutions.delete(key);
|
|
1488
|
+
});
|
|
1489
|
+
void tail.then(() => {
|
|
1490
|
+
if (commandTails.get(record.slug) === tail)
|
|
1491
|
+
commandTails.delete(record.slug);
|
|
1492
|
+
});
|
|
1493
|
+
return execution;
|
|
1494
|
+
}
|
|
1495
|
+
function acceptCommand(slug, payload, desiredState) {
|
|
1496
|
+
return fenceStore.accept(slug, commandEnvelopeFromPayload(payload), desiredState);
|
|
1497
|
+
}
|
|
1498
|
+
/**
|
|
1499
|
+
* A Start or Create for a slug whose Stop is still executing waits for that
|
|
1500
|
+
* Stop to finish before it is accepted. The Hub calls an environment
|
|
1501
|
+
* stopped as soon as its container reports the Stop, while `docker stop`
|
|
1502
|
+
* can still take seconds to return here; accepting the Start then would
|
|
1503
|
+
* cut the Stop off at its next fence check, before it records the proof
|
|
1504
|
+
* that lets this Start reuse the container. Waiting changes nothing else:
|
|
1505
|
+
* the per-slug queue would run the Start after the Stop anyway, and a
|
|
1506
|
+
* Start older than the Stop is still refused by the fence once accepted.
|
|
1507
|
+
* An inspection waits the same way, but only up to `boundMs`, and then
|
|
1508
|
+
* answers with whatever it sees.
|
|
1509
|
+
*/
|
|
1510
|
+
async function waitForInFlightStop(slug, action, boundMs) {
|
|
1511
|
+
if (!inFlightStops.has(slug))
|
|
1512
|
+
return;
|
|
1513
|
+
const startedAt = performance.now();
|
|
1514
|
+
let waited = false;
|
|
1515
|
+
let timer;
|
|
1516
|
+
const bound = boundMs === undefined
|
|
1517
|
+
? null
|
|
1518
|
+
: new Promise((resolvePromise) => { timer = setTimeout(() => resolvePromise("timed-out"), boundMs); });
|
|
1519
|
+
try {
|
|
1520
|
+
for (let stop = inFlightStops.get(slug); stop; stop = inFlightStops.get(slug)) {
|
|
1521
|
+
waited = true;
|
|
1522
|
+
if (await (bound ? Promise.race([stop, bound]) : stop) === "timed-out") {
|
|
1523
|
+
console.log(`[runner] ${action} ${slug} stopped waiting for its in-flight Stop after ${Math.round(performance.now() - startedAt)} ms`);
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
finally {
|
|
1529
|
+
if (timer)
|
|
1530
|
+
clearTimeout(timer);
|
|
1531
|
+
}
|
|
1532
|
+
if (waited) {
|
|
1533
|
+
console.log(`[runner] ${action} ${slug} waited ${Math.round(performance.now() - startedAt)} ms for its in-flight Stop to finish`);
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* The runner's backstop for the Hub's own check: refuses a Create or Start
|
|
1538
|
+
* past the environment limit, before the command is accepted. Never refuses
|
|
1539
|
+
* an isolated run, a retry of the command already accepted for the slug, a
|
|
1540
|
+
* container already running, or when the limit or the count is unknown.
|
|
1541
|
+
*/
|
|
1542
|
+
async function refuseWhenFull(slug, payload) {
|
|
1543
|
+
const limit = config.environmentLimit?.() ?? null;
|
|
1544
|
+
if (limit === null)
|
|
1545
|
+
return;
|
|
1546
|
+
const envVars = payload.envVars && typeof payload.envVars === "object"
|
|
1547
|
+
? payload.envVars
|
|
1548
|
+
: {};
|
|
1549
|
+
if (isIsolatedRun(envVars))
|
|
1550
|
+
return;
|
|
1551
|
+
const envelope = commandEnvelopeFromPayload(payload);
|
|
1552
|
+
const accepted = fenceStore.read(slug);
|
|
1553
|
+
if (accepted
|
|
1554
|
+
&& envelope.commandGeneration !== undefined
|
|
1555
|
+
&& accepted.commandGeneration === envelope.commandGeneration
|
|
1556
|
+
&& accepted.operationId === envelope.operationId)
|
|
1557
|
+
return;
|
|
1558
|
+
const count = await (config.countRunningEnvironments ?? (() => countRunningEnvironments(runCapacityDocker)))();
|
|
1559
|
+
if (!isMachineFull(count, limit))
|
|
1560
|
+
return;
|
|
1561
|
+
const info = await inspectContainer(slug).catch(() => null);
|
|
1562
|
+
if (info && getStatus(info) === "running")
|
|
1563
|
+
return;
|
|
1564
|
+
throw new MachineFullError(count, limit);
|
|
1565
|
+
}
|
|
1566
|
+
async function executeFencedStart(record, payload, action, timer = createStepTimer()) {
|
|
1567
|
+
timer.mark("queue");
|
|
1568
|
+
const guard = () => fenceStore.assertCurrent(record);
|
|
1569
|
+
const fencedPayload = {
|
|
1570
|
+
...payload,
|
|
1571
|
+
commandGeneration: record.commandGeneration,
|
|
1572
|
+
operationId: record.operationId,
|
|
1573
|
+
desiredState: record.desiredState,
|
|
1574
|
+
};
|
|
1575
|
+
const context = {
|
|
1576
|
+
guard,
|
|
1577
|
+
fenceDirectory: fenceStore.directoryFor(record.slug),
|
|
1578
|
+
readWorkspaceSyncedStop: () => fenceStore.readWorkspaceSyncedStop(record.slug),
|
|
1579
|
+
readLaunched: () => fenceStore.readLaunched(record.slug),
|
|
1580
|
+
recordLaunch: (containerId, envFileContents) => {
|
|
1581
|
+
fenceStore.writeLaunchFile(record, envFileContents);
|
|
1582
|
+
fenceStore.recordLaunched(record, containerId);
|
|
1583
|
+
},
|
|
1584
|
+
resolveGitMirror: (mirrorPayload) => resolveGitMirrorMount(mirrorPayload, localStateDir, mirrorPresence),
|
|
1585
|
+
terminalLane: config.terminalLane ?? null,
|
|
1586
|
+
resolveWorkspaceVolume: (volumePayload, state) => resolveWorkspaceVolume(record, volumePayload, state),
|
|
1587
|
+
mark: timer.mark,
|
|
1588
|
+
resolveHarnessOverlay,
|
|
1589
|
+
};
|
|
1590
|
+
// Keeps the development answer fresh for the staging delays.
|
|
1591
|
+
if (developmentKnobs)
|
|
1592
|
+
void developmentHub.isDevelopment();
|
|
1593
|
+
const result = action === "create"
|
|
1594
|
+
? await createContainer(fencedPayload, image, context)
|
|
1595
|
+
: await ensureStarted(record.slug, fencedPayload, image, context);
|
|
1596
|
+
guard();
|
|
1597
|
+
fenceStore.markApplied(record);
|
|
1598
|
+
timer.mark("mark-applied");
|
|
1599
|
+
console.log(`[runner] ${action} ${record.slug} took ${timer.line()}`);
|
|
1600
|
+
if (result.workspaceVolumeKind === "staged")
|
|
1601
|
+
refillAfterAdoption(fencedPayload);
|
|
1602
|
+
else if (result.workspaceVolumeKind)
|
|
1603
|
+
scheduleStagingAfterCreate(fencedPayload, record);
|
|
1604
|
+
return result;
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* The slug's recorded volume; otherwise, with staging on, a staged volume
|
|
1608
|
+
* adopted now or the default name, recorded before `docker create`. With
|
|
1609
|
+
* staging off nothing is recorded, so switching it off needs no migration,
|
|
1610
|
+
* while a slug that adopted a volume keeps it.
|
|
1611
|
+
*/
|
|
1612
|
+
async function resolveWorkspaceVolume(record, payload, state) {
|
|
1613
|
+
const recorded = fenceStore.readWorkspaceVolume(record.slug);
|
|
1614
|
+
if (recorded) {
|
|
1615
|
+
updateInventory((entries) => entries.recordWorkspace(record.slug, recorded));
|
|
1616
|
+
return { volume: recorded, kind: "existing" };
|
|
1617
|
+
}
|
|
1618
|
+
const fallback = workspaceVolumeName(record.slug);
|
|
1619
|
+
let kind = state.containerExists ? "existing" : "fresh";
|
|
1620
|
+
if (!stagedWorkspaces)
|
|
1621
|
+
return { volume: fallback, kind };
|
|
1622
|
+
stagedWorkspaces.noteDependencyPreference(payload.repoUrl, payload.envVars);
|
|
1623
|
+
const held = stagingHold.active();
|
|
1624
|
+
// Only a slug that has no workspace yet adopts. A legacy slug's default
|
|
1625
|
+
// volume may hold work that never reached the Hub.
|
|
1626
|
+
if (kind === "fresh" && payload.envVars.TILLER_PORTABLE_WORKSPACE_RESTORE === undefined) {
|
|
1627
|
+
if (await volumeExists(fallback)) {
|
|
1628
|
+
kind = "existing";
|
|
1629
|
+
}
|
|
1630
|
+
else if (held) {
|
|
1631
|
+
console.log(`[runner] ${record.slug} adopts no staged workspace: staging is held for the startup bench`);
|
|
1632
|
+
}
|
|
1633
|
+
else {
|
|
1634
|
+
const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
|
|
1635
|
+
const adopted = await stagedWorkspaces.adopt(payload.repoUrl, base, record.slug, (volume) => fenceStore.writeWorkspaceVolume(record.slug, volume), () => fenceStore.assertCurrent(record));
|
|
1636
|
+
if (adopted) {
|
|
1637
|
+
console.log(`[runner] ${record.slug} adopted staged workspace ${adopted.volume} at ${adopted.commit.slice(0, 12)}`
|
|
1638
|
+
+ (adopted.exact ? "" : `; the bootstrap updates it to ${base?.slice(0, 12)}`));
|
|
1639
|
+
return {
|
|
1640
|
+
volume: adopted.volume,
|
|
1641
|
+
kind: "staged",
|
|
1642
|
+
...(adopted.deps ? { stagedDependencies: adopted.deps } : {}),
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
fenceStore.assertCurrent(record);
|
|
1648
|
+
const volume = fenceStore.writeWorkspaceVolume(record.slug, fallback);
|
|
1649
|
+
updateInventory((entries) => entries.recordWorkspace(record.slug, volume));
|
|
1650
|
+
return { volume, kind: volume === fallback ? kind : "existing" };
|
|
1651
|
+
}
|
|
1652
|
+
function stageAndLog(repoUrl, commit, trigger, options = {}) {
|
|
1653
|
+
if (!stagedWorkspaces)
|
|
1654
|
+
return Promise.resolve("skipped");
|
|
1655
|
+
if (stagingHold.active()) {
|
|
1656
|
+
console.log(`[runner] not staging a workspace for ${repoUrl} (${trigger}): staging is held for the startup bench`);
|
|
1657
|
+
return Promise.resolve("skipped");
|
|
1658
|
+
}
|
|
1659
|
+
return stagedWorkspaces.stage(repoUrl, commit, { replace: options.replace }).then(({ result, done }) => {
|
|
1660
|
+
void done.then((outcome) => {
|
|
1661
|
+
if (outcome.kind === "skipped" && result === "skipped") {
|
|
1662
|
+
console.log(`[runner] not staging a workspace for ${repoUrl} (${trigger}): ${outcome.reason}`);
|
|
1663
|
+
}
|
|
1664
|
+
if (outcome.kind === "commit-missing")
|
|
1665
|
+
options.onCommitMissing?.();
|
|
1666
|
+
});
|
|
1667
|
+
return result;
|
|
1668
|
+
}, (error) => {
|
|
1669
|
+
console.error(`[runner] staging a workspace for ${repoUrl} failed (${trigger}):`, error instanceof Error ? error.message : String(error));
|
|
1670
|
+
return "skipped";
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Replaces a staged volume shortly after a create has adopted it, once the
|
|
1675
|
+
* adopting environment has had time to reach ready, so the stage run does
|
|
1676
|
+
* not slow that start. The mirror held the adopted commit, so when the base
|
|
1677
|
+
* is that commit the stage run can go ahead whatever happens to the
|
|
1678
|
+
* adopting environment: a Stop or Delete does not cancel it. When the
|
|
1679
|
+
* volume was at an older commit the mirror may not hold the base yet; that
|
|
1680
|
+
* run ends as commit-missing and the delayed path takes over.
|
|
1681
|
+
*/
|
|
1682
|
+
function refillAfterAdoption(payload) {
|
|
1683
|
+
if (!stagedWorkspaces)
|
|
1684
|
+
return;
|
|
1685
|
+
const repository = canonicalGitHubRepositoryUrl(payload.repoUrl);
|
|
1686
|
+
const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
|
|
1687
|
+
if (!repository || !base)
|
|
1688
|
+
return;
|
|
1689
|
+
const timer = setTimeout(() => {
|
|
1690
|
+
refillTimers.delete(timer);
|
|
1691
|
+
void stageAndLog(repository, base, "after adoption", {
|
|
1692
|
+
onCommitMissing: () => scheduleStagingAfterCreate(payload),
|
|
1693
|
+
});
|
|
1694
|
+
}, config.refillAfterAdoptionDelayMs ?? developmentDelay(development.refillDelayMs, REFILL_AFTER_ADOPTION_DELAY_MS));
|
|
1695
|
+
timer.unref();
|
|
1696
|
+
refillTimers.add(timer);
|
|
1697
|
+
}
|
|
1698
|
+
/**
|
|
1699
|
+
* Replenishes the repository's staged volume a minute after a create that
|
|
1700
|
+
* adopted none, once the environment's own boot has written its base into
|
|
1701
|
+
* the mirror. Creates of the same repository and base within that minute
|
|
1702
|
+
* share one timer, which stages when any of them is still current: with
|
|
1703
|
+
* `record`, a Stop or Delete of that one environment only withdraws it, and
|
|
1704
|
+
* without, nothing cancels it. When the run finds the mirror without the
|
|
1705
|
+
* base (the first environment left before its boot fetched it), environments
|
|
1706
|
+
* still current get one more full delay.
|
|
1707
|
+
*/
|
|
1708
|
+
function scheduleStagingAfterCreate(payload, record, retry = false) {
|
|
1709
|
+
if (!stagedWorkspaces)
|
|
1710
|
+
return;
|
|
1711
|
+
const repository = canonicalGitHubRepositoryUrl(payload.repoUrl);
|
|
1712
|
+
const base = payload.envVars.TILLER_GITHUB_BASE_COMMIT_SHA;
|
|
1713
|
+
if (!repository || !base)
|
|
1714
|
+
return;
|
|
1715
|
+
const key = `${repository}@${base}`;
|
|
1716
|
+
const pending = stageTimers.get(key);
|
|
1717
|
+
if (pending) {
|
|
1718
|
+
pending.records.push(record ?? null);
|
|
1719
|
+
return;
|
|
1720
|
+
}
|
|
1721
|
+
const records = [record ?? null];
|
|
1722
|
+
const timer = setTimeout(() => {
|
|
1723
|
+
stageTimers.delete(key);
|
|
1724
|
+
const live = records.filter((waiting) => {
|
|
1725
|
+
if (!waiting)
|
|
1726
|
+
return true;
|
|
1727
|
+
try {
|
|
1728
|
+
fenceStore.assertCurrent(waiting);
|
|
1729
|
+
return true;
|
|
1730
|
+
}
|
|
1731
|
+
catch {
|
|
1732
|
+
return false;
|
|
1733
|
+
}
|
|
1734
|
+
});
|
|
1735
|
+
if (!live.length) {
|
|
1736
|
+
console.log(`[runner] not staging a workspace for ${repository} (after create): every environment that asked was stopped or deleted`);
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
void stageAndLog(repository, base, "after create", {
|
|
1740
|
+
onCommitMissing: retry ? undefined : () => {
|
|
1741
|
+
for (const waiting of live)
|
|
1742
|
+
scheduleStagingAfterCreate(payload, waiting ?? undefined, true);
|
|
1743
|
+
},
|
|
1744
|
+
});
|
|
1745
|
+
}, config.stageAfterCreateDelayMs ?? developmentDelay(development.stageDelayMs, STAGE_AFTER_CREATE_DELAY_MS));
|
|
1746
|
+
timer.unref();
|
|
1747
|
+
stageTimers.set(key, { timer, records });
|
|
1748
|
+
}
|
|
1749
|
+
/**
|
|
1750
|
+
* Refreshes the repository's mirror in a short-lived root container that
|
|
1751
|
+
* holds only the Hub's scoped, read-only bridge credential, then starts
|
|
1752
|
+
* staging a workspace at that commit, in place of one staged at another
|
|
1753
|
+
* head. Concurrent requests for the same
|
|
1754
|
+
* mirror and commit share one run.
|
|
1755
|
+
*/
|
|
1756
|
+
function prepareProject(request) {
|
|
1757
|
+
const repository = canonicalGitHubRepositoryUrl(request.repoUrl);
|
|
1758
|
+
const mirrorVolume = repository ? mirrorVolumeName(repository) : null;
|
|
1759
|
+
if (!repository || !mirrorVolume) {
|
|
1760
|
+
return Promise.resolve({ ok: true, mirror: "skipped", staged: "skipped" });
|
|
1761
|
+
}
|
|
1762
|
+
const env = buildMirrorRefreshEnv(request, repository);
|
|
1763
|
+
const mirrorOnly = request.stage === false;
|
|
1764
|
+
const key = `${mirrorVolume}@${request.headSha}${mirrorOnly ? ":mirror" : ""}`;
|
|
1765
|
+
const existing = preparations.get(key);
|
|
1766
|
+
if (existing)
|
|
1767
|
+
return existing;
|
|
1768
|
+
const preparation = (async () => {
|
|
1769
|
+
try {
|
|
1770
|
+
recordGitMirrorUse(mirrorVolume, repository, localStateDir);
|
|
1771
|
+
}
|
|
1772
|
+
catch (error) {
|
|
1773
|
+
console.error(`[runner] could not record Git mirror use for ${mirrorVolume}:`, error instanceof Error ? error.message : String(error));
|
|
1774
|
+
}
|
|
1775
|
+
const envFile = createDockerEnvFile(env);
|
|
1776
|
+
const refreshName = dockerName(`mirror-refresh-${randomUUID()}`);
|
|
1777
|
+
let timedOut = false;
|
|
1778
|
+
const timer = setTimeout(() => {
|
|
1779
|
+
timedOut = true;
|
|
1780
|
+
void runDocker(["rm", "-f", refreshName]).catch(() => { });
|
|
1781
|
+
}, MIRROR_REFRESH_CONTAINER_TIMEOUT_MS);
|
|
1782
|
+
timer.unref();
|
|
1783
|
+
let output;
|
|
1784
|
+
try {
|
|
1785
|
+
output = await runDocker(buildMirrorRefreshRunArgs({
|
|
1786
|
+
image,
|
|
1787
|
+
containerName: refreshName,
|
|
1788
|
+
mirrorVolume,
|
|
1789
|
+
envFilePath: envFile.filePath,
|
|
1790
|
+
}));
|
|
1791
|
+
}
|
|
1792
|
+
catch (error) {
|
|
1793
|
+
throw redactError(timedOut ? new Error("The mirror refresh timed out") : error, Object.fromEntries(PREPARE_PROJECT_SECRET_NAMES.flatMap((name) => env[name] ? [[name, env[name]]] : [])));
|
|
1794
|
+
}
|
|
1795
|
+
finally {
|
|
1796
|
+
clearTimeout(timer);
|
|
1797
|
+
envFile.cleanup();
|
|
1798
|
+
}
|
|
1799
|
+
const mirror = parseMirrorRefreshOutcome(output);
|
|
1800
|
+
if (!mirror)
|
|
1801
|
+
throw new Error("The mirror refresh reported no outcome");
|
|
1802
|
+
// A plan's basis commit only needs to be in the mirror; the staged
|
|
1803
|
+
// workspace stays at the default-branch head.
|
|
1804
|
+
if (mirrorOnly) {
|
|
1805
|
+
console.log(`[runner] mirrored ${repository} at ${request.headSha.slice(0, 12)}: mirror=${mirror}`);
|
|
1806
|
+
return { ok: true, mirror, staged: "skipped" };
|
|
1807
|
+
}
|
|
1808
|
+
// The mirror holds the head now, so a volume staged at an older head is replaced.
|
|
1809
|
+
const staged = await stageAndLog(repository, request.headSha, "project prepare", { replace: true });
|
|
1810
|
+
console.log(`[runner] prepared ${repository} at ${request.headSha.slice(0, 12)}: mirror=${mirror} staged=${staged}`);
|
|
1811
|
+
return { ok: true, mirror, staged };
|
|
1812
|
+
})();
|
|
1813
|
+
preparations.set(key, preparation);
|
|
1814
|
+
void preparation.then(() => { if (preparations.get(key) === preparation)
|
|
1815
|
+
preparations.delete(key); }, () => { if (preparations.get(key) === preparation)
|
|
1816
|
+
preparations.delete(key); });
|
|
1817
|
+
return preparation;
|
|
1818
|
+
}
|
|
1819
|
+
// Spares isolated creates a `docker volume inspect` (see VolumePresenceCache).
|
|
1820
|
+
const mirrorPresence = new VolumePresenceCache({
|
|
1821
|
+
inspect: gitMirrorVolumeExists,
|
|
1822
|
+
known: (volume) => readRunnerInventory(localStateDir)[volume]?.kind === "mirror",
|
|
1823
|
+
});
|
|
1824
|
+
/**
|
|
1825
|
+
* The one cleanup pass over the volumes the runner manages (see
|
|
1826
|
+
* `reconcileRunnerVolumes`), at startup and then daily.
|
|
1827
|
+
*/
|
|
1828
|
+
function reconcileVolumes() {
|
|
1829
|
+
void reconcileRunnerVolumes({
|
|
1830
|
+
inventory,
|
|
1831
|
+
runDocker,
|
|
1832
|
+
workspaceRecords: () => fenceStore.listWorkspaceVolumeRecords(),
|
|
1833
|
+
defaultWorkspaceVolume: workspaceVolumeName,
|
|
1834
|
+
mirrors: true,
|
|
1835
|
+
staged: stagedWorkspaces
|
|
1836
|
+
? { isBuilding: (volume) => stagedWorkspaces.isBuilding(volume), budgetBytes }
|
|
1837
|
+
: null,
|
|
1838
|
+
}).then(({ removed }) => {
|
|
1839
|
+
mirrorPresence.forget(removed);
|
|
1840
|
+
if (removed.length)
|
|
1841
|
+
console.log(`[runner] removed unused runner volumes: ${removed.join(", ")}`);
|
|
1842
|
+
}, (error) => {
|
|
1843
|
+
console.error("[runner] runner volume cleanup failed:", error instanceof Error ? error.message : String(error));
|
|
1844
|
+
});
|
|
1845
|
+
}
|
|
1846
|
+
async function executeFencedStop(record, onInspected) {
|
|
1847
|
+
closeTerminalLanes(record.slug);
|
|
1848
|
+
fenceStore.assertCurrent(record);
|
|
1849
|
+
const info = await inspectContainer(record.slug, true);
|
|
1850
|
+
fenceStore.assertCurrent(record);
|
|
1851
|
+
const launched = fenceStore.readLaunched(record.slug);
|
|
1852
|
+
if (neverStarted(info)) {
|
|
1853
|
+
// A create superseded between `docker create` and `docker start` (a
|
|
1854
|
+
// Stop sent about a second after the create returned), or a runner that
|
|
1855
|
+
// died there: Docker made the container but no process in it ever ran,
|
|
1856
|
+
// so it never touched the workspace and there is nothing to save. The
|
|
1857
|
+
// container is left as it is: the next Start's create removes a
|
|
1858
|
+
// `created` container no current command claims, and a Delete removes
|
|
1859
|
+
// it with the rest.
|
|
1860
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
1861
|
+
fenceStore.markApplied(record);
|
|
1862
|
+
onInspected({ callbackExpected: false, startRejectedBeforeWorkspace: true });
|
|
1863
|
+
return;
|
|
1864
|
+
}
|
|
1865
|
+
if (!info || getStatus(info) !== "running") {
|
|
1866
|
+
if (info && !["stopped", "exited", "dead"].includes(getStatus(info))) {
|
|
1867
|
+
throw new Error("The container is not confirmed stopped. Its retained data was left untouched.");
|
|
1868
|
+
}
|
|
1869
|
+
// Either the entrypoint refused the superseded start before touching
|
|
1870
|
+
// the workspace, or the Stop superseded the slug's first start before
|
|
1871
|
+
// Docker created a container (a Stop sent right after the create
|
|
1872
|
+
// returns): no container was ever launched here, so nothing can hold
|
|
1873
|
+
// unsaved work.
|
|
1874
|
+
const startRejectedBeforeWorkspace = info
|
|
1875
|
+
? info.State?.ExitCode === ENTRYPOINT_FENCED_BEFORE_WORKSPACE_EXIT_CODE
|
|
1876
|
+
&& launchedGeneration(info, launched) === record.commandGeneration - 1
|
|
1877
|
+
: launched === null;
|
|
1878
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
1879
|
+
fenceStore.markApplied(record);
|
|
1880
|
+
onInspected({
|
|
1881
|
+
callbackExpected: false,
|
|
1882
|
+
...(startRejectedBeforeWorkspace ? { startRejectedBeforeWorkspace: true } : {}),
|
|
1883
|
+
});
|
|
1884
|
+
return;
|
|
1885
|
+
}
|
|
1886
|
+
const bootstrapModes = (info.Config?.Env ?? []).filter(value => value.startsWith("TILLER_BOOTSTRAP_MODE="));
|
|
1887
|
+
if (bootstrapModes.length === 1 && bootstrapModes[0] === "TILLER_BOOTSTRAP_MODE=plan-writer") {
|
|
1888
|
+
if (launchedGeneration(info, launched) !== record.commandGeneration - 1)
|
|
1889
|
+
throw new Error("Scribe retention stop does not match its launched container generation");
|
|
1890
|
+
await runDocker(["stop", "-t", String(SCRIBE_STOP_GRACE_SECONDS), containerName(record.slug)]);
|
|
1891
|
+
fenceStore.assertCurrent(record);
|
|
1892
|
+
const stopped = await inspectContainer(record.slug, true);
|
|
1893
|
+
fenceStore.assertCurrent(record);
|
|
1894
|
+
if (!stopped || !["stopped", "exited", "dead"].includes(getStatus(stopped)))
|
|
1895
|
+
throw new Error("Scribe stopped-container retention was not confirmed");
|
|
1896
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
1897
|
+
fenceStore.markApplied(record);
|
|
1898
|
+
onInspected({ callbackExpected: false });
|
|
1899
|
+
return;
|
|
1900
|
+
}
|
|
1901
|
+
// Fence first, acknowledge after: once the Hub hears the Stop was
|
|
1902
|
+
// accepted, no agent can be spawned and nothing reports ready.
|
|
1903
|
+
const fence = await fenceStopInContainer(record.slug, record.operationId);
|
|
1904
|
+
fenceStore.assertCurrent(record);
|
|
1905
|
+
if (fence === "unfenced")
|
|
1906
|
+
console.error(`[runner] could not fence ${record.slug} before acknowledging the Stop; preparing it anyway`);
|
|
1907
|
+
onInspected({ callbackExpected: true });
|
|
1908
|
+
const { outcome: preparation, detail: preparationDetail } = await prepareDurableStop(record.slug, record.operationId, {
|
|
1909
|
+
idleClaimId: idleStopClaims.get(commandKey(record)) ?? null,
|
|
1910
|
+
});
|
|
1911
|
+
fenceStore.assertCurrent(record);
|
|
1912
|
+
const runningGeneration = launchedGeneration(info, launched);
|
|
1913
|
+
if (preparation !== "prepared") {
|
|
1914
|
+
// Nothing saved the workspace through stop-control: the Stop came
|
|
1915
|
+
// before the harness was let go, or stop-control cannot be reached.
|
|
1916
|
+
// Either way an accepted Stop must not leave an agent running, so the
|
|
1917
|
+
// container is stopped outright. Its entrypoint never uploads a
|
|
1918
|
+
// partial workspace; before release it confirms to the Hub that the
|
|
1919
|
+
// Hub's copy is the whole workspace and exits with a dedicated code.
|
|
1920
|
+
console.error(`[runner] stop preparation for ${record.slug} was ${preparation}${preparationDetail ? ` (${preparationDetail})` : ""}; stopping the container outright`);
|
|
1921
|
+
await runDocker(["stop", "-t", String(OUTRIGHT_STOP_GRACE_SECONDS), containerName(record.slug)]);
|
|
1922
|
+
fenceStore.assertCurrent(record);
|
|
1923
|
+
const stopped = await inspectContainer(record.slug, true);
|
|
1924
|
+
fenceStore.assertCurrent(record);
|
|
1925
|
+
if (stopped && !["stopped", "exited", "dead"].includes(getStatus(stopped))) {
|
|
1926
|
+
throw new Error("Docker did not confirm the container stopped.");
|
|
1927
|
+
}
|
|
1928
|
+
// Only a container that ended the Stop before its harness ran, and had
|
|
1929
|
+
// the Hub accept that, proves nothing is left to save; the next Start
|
|
1930
|
+
// may then replace it. Anything else stays for the user to decide.
|
|
1931
|
+
// A container that saved for itself after the preparation failed
|
|
1932
|
+
// (exit 79) proves the same. The exit code is the entrypoint's, which
|
|
1933
|
+
// the workload cannot set.
|
|
1934
|
+
const sameContainer = !info.Id || stopped?.Id === info.Id;
|
|
1935
|
+
const confirmedBeforeRelease = sameContainer && ((preparation === "stopped-before-release"
|
|
1936
|
+
&& stopped?.State?.ExitCode === ENTRYPOINT_STOPPED_BEFORE_RELEASE_EXIT_CODE)
|
|
1937
|
+
|| stopped?.State?.ExitCode === ENTRYPOINT_STOP_SAVED_BY_CLEANUP_EXIT_CODE);
|
|
1938
|
+
if (confirmedBeforeRelease && (runningGeneration !== null || info.Id)) {
|
|
1939
|
+
fenceStore.recordWorkspaceSyncedStop(record, runningGeneration, info.Id);
|
|
1940
|
+
}
|
|
1941
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
1942
|
+
fenceStore.markApplied(record);
|
|
1943
|
+
if (!confirmedBeforeRelease) {
|
|
1944
|
+
throw new Error(`The container for ${record.slug} was stopped without a confirmed workspace save (${preparation}${preparationDetail ? `: ${preparationDetail}` : ""}).`);
|
|
1945
|
+
}
|
|
1946
|
+
return;
|
|
1947
|
+
}
|
|
1948
|
+
// A durable stop of this container is proof for this container even when
|
|
1949
|
+
// the launch record no longer attributes it, so the next Start can still
|
|
1950
|
+
// replace it instead of wedging.
|
|
1951
|
+
if (runningGeneration !== null || info.Id) {
|
|
1952
|
+
fenceStore.recordWorkspaceSyncedStop(record, runningGeneration, info.Id);
|
|
1953
|
+
}
|
|
1954
|
+
await runDocker(["stop", "-t", String(PREPARED_STOP_GRACE_SECONDS), containerName(record.slug)]);
|
|
1955
|
+
fenceStore.assertCurrent(record);
|
|
1956
|
+
const stopped = await inspectContainer(record.slug, true);
|
|
1957
|
+
fenceStore.assertCurrent(record);
|
|
1958
|
+
if (stopped && !["stopped", "exited", "dead"].includes(getStatus(stopped))) {
|
|
1959
|
+
throw new Error("Docker did not confirm the container stopped.");
|
|
1960
|
+
}
|
|
1961
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
1962
|
+
fenceStore.markApplied(record);
|
|
1963
|
+
}
|
|
1964
|
+
function scheduleFencedStop(record) {
|
|
1965
|
+
const key = commandKey(record);
|
|
1966
|
+
const existing = stopAcknowledgements.get(key);
|
|
1967
|
+
if (existing)
|
|
1968
|
+
return existing;
|
|
1969
|
+
let acknowledgementSettled = false;
|
|
1970
|
+
let resolveAcknowledgement;
|
|
1971
|
+
let rejectAcknowledgement;
|
|
1972
|
+
const acknowledgement = new Promise((resolvePromise, rejectPromise) => {
|
|
1973
|
+
resolveAcknowledgement = resolvePromise;
|
|
1974
|
+
rejectAcknowledgement = rejectPromise;
|
|
1975
|
+
});
|
|
1976
|
+
// Recovery schedules Stop without an HTTP waiter; keep a rejected early
|
|
1977
|
+
// inspection from becoming an unhandled promise while preserving it for
|
|
1978
|
+
// request callers that do await the acknowledgement.
|
|
1979
|
+
void acknowledgement.catch(() => { });
|
|
1980
|
+
const completion = scheduleCommand(record, () => executeFencedStop(record, (result) => {
|
|
1981
|
+
if (acknowledgementSettled)
|
|
1982
|
+
return;
|
|
1983
|
+
acknowledgementSettled = true;
|
|
1984
|
+
resolveAcknowledgement(result);
|
|
1985
|
+
}));
|
|
1986
|
+
const scheduled = { acknowledgement, completion };
|
|
1987
|
+
stopAcknowledgements.set(key, scheduled);
|
|
1988
|
+
const settledStop = completion.then(() => undefined, () => undefined);
|
|
1989
|
+
inFlightStops.set(record.slug, settledStop);
|
|
1990
|
+
void settledStop.then(() => {
|
|
1991
|
+
if (inFlightStops.get(record.slug) === settledStop)
|
|
1992
|
+
inFlightStops.delete(record.slug);
|
|
1993
|
+
});
|
|
1994
|
+
void completion.then(() => {
|
|
1995
|
+
if (!acknowledgementSettled) {
|
|
1996
|
+
acknowledgementSettled = true;
|
|
1997
|
+
resolveAcknowledgement({ callbackExpected: false });
|
|
1998
|
+
}
|
|
1999
|
+
if (stopAcknowledgements.get(key) === scheduled)
|
|
2000
|
+
stopAcknowledgements.delete(key);
|
|
2001
|
+
}, (error) => {
|
|
2002
|
+
if (!acknowledgementSettled) {
|
|
2003
|
+
acknowledgementSettled = true;
|
|
2004
|
+
rejectAcknowledgement(error);
|
|
2005
|
+
}
|
|
2006
|
+
if (stopAcknowledgements.get(key) === scheduled)
|
|
2007
|
+
stopAcknowledgements.delete(key);
|
|
2008
|
+
});
|
|
2009
|
+
return scheduled;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* An idle Stop is decided before it is accepted: the harness must still
|
|
2013
|
+
* hold the claim (see fenceIdleStopInContainer). A refusal accepts nothing,
|
|
2014
|
+
* so the Hub can put the session back to running. A retry of a Stop this
|
|
2015
|
+
* runner already accepted is not asked again, and a container that is not
|
|
2016
|
+
* running has nothing to keep: its Stop goes ahead as any other.
|
|
2017
|
+
*/
|
|
2018
|
+
async function admitIdleStop(slug, payload, idleClaimId) {
|
|
2019
|
+
const envelope = commandEnvelopeFromPayload(payload);
|
|
2020
|
+
const operationId = typeof envelope.operationId === "string" ? envelope.operationId.trim() : "";
|
|
2021
|
+
const current = fenceStore.read(slug);
|
|
2022
|
+
if (current && current.commandGeneration === envelope.commandGeneration && current.operationId === operationId)
|
|
2023
|
+
return true;
|
|
2024
|
+
if (!operationId)
|
|
2025
|
+
return false;
|
|
2026
|
+
const info = await inspectContainer(slug, true);
|
|
2027
|
+
if (!info || getStatus(info) !== "running")
|
|
2028
|
+
return true;
|
|
2029
|
+
const decided = await fenceIdleStopInContainer(slug, operationId, idleClaimId);
|
|
2030
|
+
console.log(`[runner] idle stop for ${slug} ${decided === "fenced" ? "decided: the harness still held its claim" : "refused: the session is active or its claim is gone"}`);
|
|
2031
|
+
return decided === "fenced";
|
|
2032
|
+
}
|
|
2033
|
+
async function executeFencedDestroy(record) {
|
|
2034
|
+
fenceStore.assertCurrent(record);
|
|
2035
|
+
closeTerminalLanes(record.slug);
|
|
2036
|
+
const removed = await forceRemoveContainer(record.slug);
|
|
2037
|
+
fenceStore.assertCurrent(record);
|
|
2038
|
+
if (await inspectContainer(record.slug, true)) {
|
|
2039
|
+
throw new Error("Docker did not confirm the container was removed.");
|
|
2040
|
+
}
|
|
2041
|
+
let recordedVolume = null;
|
|
2042
|
+
try {
|
|
2043
|
+
recordedVolume = fenceStore.readWorkspaceVolume(record.slug);
|
|
2044
|
+
}
|
|
2045
|
+
catch (error) {
|
|
2046
|
+
// The default volume still goes; an unreadable record's staged volume
|
|
2047
|
+
// is left for the staged-workspace cleanup.
|
|
2048
|
+
console.error(`[runner] ${record.slug}:`, error instanceof Error ? error.message : String(error));
|
|
2049
|
+
}
|
|
2050
|
+
const removedWorkspaceVolume = await removeWorkspaceVolumes(record.slug, recordedVolume, image);
|
|
2051
|
+
fenceStore.assertCurrent(record);
|
|
2052
|
+
fenceStore.removeWorkspaceVolume(record.slug);
|
|
2053
|
+
updateInventory((entries) => entries.forgetWorkspace(record.slug));
|
|
2054
|
+
const removedOpencodeState = await removeLocalOpencodeStateForRunner(record.slug, image);
|
|
2055
|
+
fenceStore.assertCurrent(record);
|
|
2056
|
+
fenceStore.removeLaunchFiles(record.slug);
|
|
2057
|
+
fenceStore.removeLaunched(record.slug);
|
|
2058
|
+
fenceStore.markApplied(record);
|
|
2059
|
+
return { removed: removed || removedWorkspaceVolume || removedOpencodeState };
|
|
2060
|
+
}
|
|
2061
|
+
function recoverAcceptedTerminalCommands() {
|
|
2062
|
+
for (const record of fenceStore.listAcceptedTerminalCommands()) {
|
|
2063
|
+
const execution = record.desiredState === "stopped"
|
|
2064
|
+
? scheduleFencedStop(record).completion
|
|
2065
|
+
: scheduleCommand(record, () => executeFencedDestroy(record));
|
|
2066
|
+
void execution.catch((error) => {
|
|
2067
|
+
console.error(`[runner] Failed to recover ${record.desiredState} command for ${record.slug}:`, error instanceof Error ? error.message : String(error));
|
|
2068
|
+
});
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
/**
|
|
2072
|
+
* Development only: `POST /dev/staging-hold` with `{ "hold": true, "ms": N }`
|
|
2073
|
+
* holds staging for N ms (default 10 min, at most an hour), `{ "hold": false }`
|
|
2074
|
+
* ends it. The startup bench holds it so `mirror-hit` starts never adopt.
|
|
2075
|
+
*/
|
|
2076
|
+
async function handleStagingHold(req, res) {
|
|
2077
|
+
if (req.method !== "POST") {
|
|
2078
|
+
methodNotAllowed(res);
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
try {
|
|
2082
|
+
const body = await readJson(req);
|
|
2083
|
+
if (!(await developmentHub.isDevelopment())) {
|
|
2084
|
+
json(res, 403, { ok: false, error: "The staging hold is for development Hubs only." });
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
if (body.hold === false) {
|
|
2088
|
+
stagingHold.release();
|
|
2089
|
+
console.log("[runner] staging hold ended");
|
|
2090
|
+
json(res, 200, { ok: true, held: false, staging: stagedWorkspaces ? "on" : "off" });
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
if (body.hold !== true || (body.ms !== undefined && (!Number.isSafeInteger(body.ms) || body.ms < 0))) {
|
|
2094
|
+
json(res, 400, { ok: false, error: "Expected { \"hold\": true, \"ms\"?: <non-negative integer> } or { \"hold\": false }." });
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
2097
|
+
const until = stagingHold.hold(body.ms ?? STAGING_HOLD_DEFAULT_MS);
|
|
2098
|
+
console.log(`[runner] staging held for the startup bench until ${new Date(until).toISOString()}`);
|
|
2099
|
+
json(res, 200, { ok: true, held: true, until: new Date(until).toISOString(), staging: stagedWorkspaces ? "on" : "off" });
|
|
2100
|
+
}
|
|
2101
|
+
catch (error) {
|
|
2102
|
+
requestError(res, error);
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
async function handleApi(req, res) {
|
|
2106
|
+
const url = new URL(req.url, `http://127.0.0.1:${port}`);
|
|
2107
|
+
if (url.pathname === "/healthz") {
|
|
2108
|
+
json(res, 200, {
|
|
2109
|
+
ok: true,
|
|
2110
|
+
identity,
|
|
2111
|
+
capabilities: runnerCapabilities,
|
|
2112
|
+
});
|
|
2113
|
+
return;
|
|
2114
|
+
}
|
|
2115
|
+
// Everything past the health check needs the API secret: session
|
|
2116
|
+
// containers reach this port through `host.docker.internal`.
|
|
2117
|
+
if (!isAuthorizedRunnerRequest(req, apiToken)) {
|
|
2118
|
+
res.setHeader("WWW-Authenticate", "Bearer");
|
|
2119
|
+
json(res, 401, { error: "Unauthorized" });
|
|
2120
|
+
return;
|
|
2121
|
+
}
|
|
2122
|
+
if (isRefusedBrowserMutation(req)) {
|
|
2123
|
+
json(res, 403, { error: "Forbidden" });
|
|
2124
|
+
return;
|
|
2125
|
+
}
|
|
2126
|
+
// Every route under /envs/ names an environment in its path; the slug
|
|
2127
|
+
// becomes Docker names and a host folder, so it must be a plain name.
|
|
2128
|
+
const envSegment = url.pathname.match(/^\/envs\/([^/]+)/);
|
|
2129
|
+
if (envSegment) {
|
|
2130
|
+
let slug = null;
|
|
2131
|
+
try {
|
|
2132
|
+
slug = decodeURIComponent(envSegment[1]);
|
|
2133
|
+
}
|
|
2134
|
+
catch {
|
|
2135
|
+
// Malformed escapes are refused below like any other bad slug.
|
|
2136
|
+
}
|
|
2137
|
+
if (!isValidRunnerSlug(slug)) {
|
|
2138
|
+
requestError(res, new RunnerSlugError());
|
|
2139
|
+
return;
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
if (url.pathname === "/dev/staging-hold") {
|
|
2143
|
+
await handleStagingHold(req, res);
|
|
2144
|
+
return;
|
|
2145
|
+
}
|
|
2146
|
+
if (url.pathname === "/projects/prepare") {
|
|
2147
|
+
if (req.method !== "POST") {
|
|
2148
|
+
methodNotAllowed(res);
|
|
2149
|
+
return;
|
|
2150
|
+
}
|
|
2151
|
+
let request = null;
|
|
2152
|
+
try {
|
|
2153
|
+
request = parseProjectPreparationRequest(await readJson(req).catch((error) => {
|
|
2154
|
+
throw new ProjectPreparationRequestError(error instanceof Error ? error.message : String(error));
|
|
2155
|
+
}));
|
|
2156
|
+
json(res, 200, await prepareProject(request));
|
|
2157
|
+
}
|
|
2158
|
+
catch (error) {
|
|
2159
|
+
const secrets = Object.fromEntries(PREPARE_PROJECT_SECRET_NAMES.flatMap((name) => request?.envVars[name] ? [[name, request.envVars[name]]] : []));
|
|
2160
|
+
const message = redactEnvValues(error instanceof Error ? error.message : String(error), secrets);
|
|
2161
|
+
json(res, error instanceof ProjectPreparationRequestError ? 400 : 502, { ok: false, error: message });
|
|
2162
|
+
}
|
|
2163
|
+
return;
|
|
2164
|
+
}
|
|
2165
|
+
if (url.pathname === "/gadget-jobs") {
|
|
2166
|
+
// The build-and-test job for one app (gadget-job.ts). Like every route
|
|
2167
|
+
// past the health check it needs the API secret, which no container
|
|
2168
|
+
// has, and it is refused when it comes from a browser. The request
|
|
2169
|
+
// names no mount, port, image or variable: any field but the app's id,
|
|
2170
|
+
// a commit, files and a catalog is refused.
|
|
2171
|
+
if (req.method !== "POST") {
|
|
2172
|
+
methodNotAllowed(res);
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
if (!runnerCapabilities.features?.includes("gadget-job")) {
|
|
2176
|
+
json(res, 409, { ok: false, code: "gadget_job_unsupported", error: "This machine's image does not carry the build-and-test job. Update the machine." });
|
|
2177
|
+
return;
|
|
2178
|
+
}
|
|
2179
|
+
try {
|
|
2180
|
+
json(res, 200, await gadgetJobs.run(parseGadgetJobRequest(await readGadgetJobBody(req))));
|
|
2181
|
+
}
|
|
2182
|
+
catch (error) {
|
|
2183
|
+
if (error instanceof GadgetJobRequestError)
|
|
2184
|
+
json(res, error.status, { ok: false, code: error.code, error: error.message });
|
|
2185
|
+
else
|
|
2186
|
+
json(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
2187
|
+
}
|
|
2188
|
+
return;
|
|
2189
|
+
}
|
|
2190
|
+
if (url.pathname === "/gadget-map-backs") {
|
|
2191
|
+
// The same job's second mode: mapping an OS edit back onto an app's source, for a pull.
|
|
2192
|
+
// The same rules as above; the request may carry the app's id, a commit, the app's files
|
|
2193
|
+
// and the OS's files and nothing else.
|
|
2194
|
+
if (req.method !== "POST") {
|
|
2195
|
+
methodNotAllowed(res);
|
|
2196
|
+
return;
|
|
2197
|
+
}
|
|
2198
|
+
if (!runnerCapabilities.features?.includes("gadget-map-back")) {
|
|
2199
|
+
json(res, 409, { ok: false, code: "gadget_job_unsupported", error: `This machine's image does not map ${OS.noun} edits back. Update the machine.` });
|
|
2200
|
+
return;
|
|
2201
|
+
}
|
|
2202
|
+
try {
|
|
2203
|
+
json(res, 200, await gadgetJobs.mapBack(parseGadgetMapBackRequest(await readGadgetJobBody(req))));
|
|
2204
|
+
}
|
|
2205
|
+
catch (error) {
|
|
2206
|
+
if (error instanceof GadgetJobRequestError)
|
|
2207
|
+
json(res, error.status, { ok: false, code: error.code, error: error.message });
|
|
2208
|
+
else
|
|
2209
|
+
json(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
2210
|
+
}
|
|
2211
|
+
return;
|
|
2212
|
+
}
|
|
2213
|
+
const laneTicketMatch = url.pathname.match(/^\/envs\/([^/]+)\/terminal-lane-tickets$/);
|
|
2214
|
+
if (laneTicketMatch) {
|
|
2215
|
+
// Minted only for the runner's own Hub session client; the API secret
|
|
2216
|
+
// and the browser guard at the top of handleApi refuse everything else.
|
|
2217
|
+
if (req.method !== "POST") {
|
|
2218
|
+
methodNotAllowed(res);
|
|
2219
|
+
return;
|
|
2220
|
+
}
|
|
2221
|
+
if (!config.terminalLane) {
|
|
2222
|
+
json(res, 404, { error: "This runner offers no terminal lane" });
|
|
2223
|
+
return;
|
|
2224
|
+
}
|
|
2225
|
+
try {
|
|
2226
|
+
const request = parseTerminalLaneTicketRequest(decodeURIComponent(laneTicketMatch[1]), await readJson(req));
|
|
2227
|
+
const inspected = await runDocker([
|
|
2228
|
+
"inspect", "--type", "container", "-f", TERMINAL_LANE_INSPECT_FORMAT, containerName(request.slug),
|
|
2229
|
+
]).catch(() => "");
|
|
2230
|
+
if (!hasRunningTerminalLane(inspected)) {
|
|
2231
|
+
json(res, 409, { error: "The environment has no terminal lane" });
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
const address = server.address();
|
|
2235
|
+
const listeningPort = address && typeof address === "object" ? address.port : port;
|
|
2236
|
+
json(res, 200, { ...terminalLaneTickets.mint(request), port: listeningPort });
|
|
2237
|
+
}
|
|
2238
|
+
catch (error) {
|
|
2239
|
+
json(res, 400, { error: error instanceof Error ? error.message : String(error) });
|
|
2240
|
+
}
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
if (req.method === "POST" && url.pathname === "/envs") {
|
|
2244
|
+
try {
|
|
2245
|
+
const timer = createStepTimer();
|
|
2246
|
+
const payload = await readJson(req);
|
|
2247
|
+
timer.mark("read-body");
|
|
2248
|
+
const createPayload = payload;
|
|
2249
|
+
assertValidRunnerSlug(createPayload.slug);
|
|
2250
|
+
await waitForInFlightStop(createPayload.slug, "create");
|
|
2251
|
+
await refuseWhenFull(createPayload.slug, payload);
|
|
2252
|
+
const record = acceptCommand(createPayload.slug, payload, "running");
|
|
2253
|
+
timer.mark("accept");
|
|
2254
|
+
const created = record.phase === "applied"
|
|
2255
|
+
? await inspectAppliedRunningCommand(record, fenceStore, createPayload.envVars)
|
|
2256
|
+
: await scheduleCommand(record, () => executeFencedStart(record, createPayload, "create", timer));
|
|
2257
|
+
json(res, 201, { ...created, ...commandResponse(record) });
|
|
2258
|
+
}
|
|
2259
|
+
catch (err) {
|
|
2260
|
+
requestError(res, err);
|
|
2261
|
+
}
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
const inspectionMatch = url.pathname.match(/^\/envs\/([^/]+)\/inspection$/);
|
|
2265
|
+
if (req.method === "GET" && inspectionMatch) {
|
|
2266
|
+
try {
|
|
2267
|
+
const slug = decodeURIComponent(inspectionMatch[1]);
|
|
2268
|
+
// The Hub inspects before it dispatches a Start, and it may call the
|
|
2269
|
+
// environment stopped (the container reported its Stop) while
|
|
2270
|
+
// `docker stop` is still returning here. Answering then shows a
|
|
2271
|
+
// container still running, or the Stop's own fence write landing
|
|
2272
|
+
// mid-inspection (`command-changed`), and the Start is refused. So,
|
|
2273
|
+
// like create and start, wait briefly for the Stop to finish. A
|
|
2274
|
+
// container running with no Stop in flight is answered at once.
|
|
2275
|
+
await waitForInFlightStop(slug, "inspection", config.inspectionStopWaitMs ?? INSPECTION_STOP_WAIT_MS);
|
|
2276
|
+
const before = fenceStore.read(slug);
|
|
2277
|
+
const info = await inspectContainer(slug, true);
|
|
2278
|
+
const after = fenceStore.read(slug);
|
|
2279
|
+
const launched = fenceStore.readLaunched(slug);
|
|
2280
|
+
let persistence = null;
|
|
2281
|
+
try {
|
|
2282
|
+
persistence = fenceStore.readWorkspaceSyncedStop(slug);
|
|
2283
|
+
}
|
|
2284
|
+
catch {
|
|
2285
|
+
// An unreadable proof proves nothing; the inspection goes on without it.
|
|
2286
|
+
}
|
|
2287
|
+
json(res, 200, inspectRunnerState(slug, info ? getStatus(info) : null, before, after, failedBeforeHarnessProof(info, launched), !launched, durableStopProof(info, persistence)));
|
|
2288
|
+
}
|
|
2289
|
+
catch (error) {
|
|
2290
|
+
requestError(res, error, 500);
|
|
2291
|
+
}
|
|
2292
|
+
return;
|
|
2293
|
+
}
|
|
2294
|
+
const envMatch = url.pathname.match(/^\/envs\/([^/]+)$/);
|
|
2295
|
+
if (req.method === "GET" && envMatch) {
|
|
2296
|
+
const slug = decodeURIComponent(envMatch[1]);
|
|
2297
|
+
const info = await inspectContainer(slug);
|
|
2298
|
+
if (!info) {
|
|
2299
|
+
notFound(res);
|
|
2300
|
+
return;
|
|
2301
|
+
}
|
|
2302
|
+
const launched = fenceStore.readLaunched(slug);
|
|
2303
|
+
const preHarnessFailure = failedBeforeHarnessProof(info, launched);
|
|
2304
|
+
// The Hub knows this machine's pinned image, never a container's, so the
|
|
2305
|
+
// host is the only place a pending recreate can be detected.
|
|
2306
|
+
const pinnedImageId = await resolveImageId(image);
|
|
2307
|
+
const recreatePending = Boolean(pinnedImageId && info.Image && info.Image !== pinnedImageId);
|
|
2308
|
+
json(res, 200, {
|
|
2309
|
+
runnerId: containerName(slug),
|
|
2310
|
+
status: getStatus(info),
|
|
2311
|
+
...(recreatePending ? { recreatePending: true } : {}),
|
|
2312
|
+
...(preHarnessFailure ? { failedStartBeforeHarness: true, ...preHarnessFailure } : {}),
|
|
2313
|
+
});
|
|
2314
|
+
return;
|
|
2315
|
+
}
|
|
2316
|
+
if (req.method === "DELETE" && envMatch) {
|
|
2317
|
+
const slug = decodeURIComponent(envMatch[1]);
|
|
2318
|
+
try {
|
|
2319
|
+
const payload = await readJson(req).catch(() => ({}));
|
|
2320
|
+
const record = acceptCommand(slug, payload, "absent");
|
|
2321
|
+
const result = await scheduleCommand(record, () => executeFencedDestroy(record));
|
|
2322
|
+
json(res, 200, {
|
|
2323
|
+
ok: true,
|
|
2324
|
+
runnerId: containerName(slug),
|
|
2325
|
+
removed: result.removed,
|
|
2326
|
+
...commandResponse(record),
|
|
2327
|
+
});
|
|
2328
|
+
}
|
|
2329
|
+
catch (error) {
|
|
2330
|
+
requestError(res, error);
|
|
2331
|
+
}
|
|
2332
|
+
return;
|
|
2333
|
+
}
|
|
2334
|
+
const idleMatch = url.pathname.match(/^\/envs\/([^/]+)\/idle-(prepare|release)$/);
|
|
2335
|
+
if (idleMatch) {
|
|
2336
|
+
if (req.method !== "POST") {
|
|
2337
|
+
methodNotAllowed(res);
|
|
2338
|
+
return;
|
|
2339
|
+
}
|
|
2340
|
+
const slug = decodeURIComponent(idleMatch[1]);
|
|
2341
|
+
try {
|
|
2342
|
+
const payload = await readJson(req).catch(() => ({}));
|
|
2343
|
+
if (idleMatch[2] === "release") {
|
|
2344
|
+
const claimId = typeof payload.claimId === "string" ? payload.claimId.trim() : "";
|
|
2345
|
+
if (!claimId)
|
|
2346
|
+
throw new Error("claimId is required");
|
|
2347
|
+
await releaseIdleStopInContainer(slug, claimId).catch((error) => {
|
|
2348
|
+
console.error(`[runner] idle claim release for ${slug} failed (it expires on its own):`, error instanceof Error ? error.message : String(error));
|
|
2349
|
+
});
|
|
2350
|
+
json(res, 200, { ok: true });
|
|
2351
|
+
return;
|
|
2352
|
+
}
|
|
2353
|
+
const idleTimeoutMs = Number(payload.idleTimeoutMs);
|
|
2354
|
+
if (!Number.isSafeInteger(idleTimeoutMs) || idleTimeoutMs <= 0)
|
|
2355
|
+
throw new Error("idleTimeoutMs must be a positive integer");
|
|
2356
|
+
const claimTtlMs = Number.isSafeInteger(payload.claimTtlMs) && Number(payload.claimTtlMs) > 0
|
|
2357
|
+
? Number(payload.claimTtlMs)
|
|
2358
|
+
: HOST_IDLE_CLAIM_TTL_MS;
|
|
2359
|
+
const info = await inspectContainer(slug, true);
|
|
2360
|
+
if (!info || getStatus(info) !== "running") {
|
|
2361
|
+
json(res, 200, { ok: true, eligible: false, reason: "not_running", remainingIdleMs: idleTimeoutMs });
|
|
2362
|
+
return;
|
|
2363
|
+
}
|
|
2364
|
+
const startedAt = performance.now();
|
|
2365
|
+
const result = await prepareIdleStopInContainer(slug, idleTimeoutMs, claimTtlMs);
|
|
2366
|
+
console.log(`[runner] idle stop preparation for ${slug}: ${result.reason} in ${Math.round(performance.now() - startedAt)} ms`);
|
|
2367
|
+
json(res, 200, { ok: true, ...result });
|
|
2368
|
+
}
|
|
2369
|
+
catch (error) {
|
|
2370
|
+
requestError(res, error, 500);
|
|
2371
|
+
}
|
|
2372
|
+
return;
|
|
2373
|
+
}
|
|
2374
|
+
const actionMatch = url.pathname.match(/^\/envs\/([^/]+)\/(start|stop)$/);
|
|
2375
|
+
if (actionMatch) {
|
|
2376
|
+
const slug = decodeURIComponent(actionMatch[1]);
|
|
2377
|
+
const action = actionMatch[2];
|
|
2378
|
+
if (req.method !== "POST") {
|
|
2379
|
+
methodNotAllowed(res);
|
|
2380
|
+
return;
|
|
2381
|
+
}
|
|
2382
|
+
if (action === "start") {
|
|
2383
|
+
try {
|
|
2384
|
+
const payload = await readJson(req);
|
|
2385
|
+
const startPayload = { ...payload, slug };
|
|
2386
|
+
await waitForInFlightStop(slug, "start");
|
|
2387
|
+
await refuseWhenFull(slug, payload);
|
|
2388
|
+
const record = acceptCommand(slug, payload, "running");
|
|
2389
|
+
const started = record.phase === "applied"
|
|
2390
|
+
? await inspectAppliedRunningCommand(record, fenceStore, startPayload.envVars)
|
|
2391
|
+
: await scheduleCommand(record, () => executeFencedStart(record, startPayload, "start"));
|
|
2392
|
+
json(res, 200, { ...started, ...commandResponse(record) });
|
|
2393
|
+
}
|
|
2394
|
+
catch (err) {
|
|
2395
|
+
requestError(res, err);
|
|
2396
|
+
}
|
|
2397
|
+
return;
|
|
2398
|
+
}
|
|
2399
|
+
if (action === "stop") {
|
|
2400
|
+
try {
|
|
2401
|
+
const payload = await readJson(req).catch(() => ({}));
|
|
2402
|
+
const idleClaimId = typeof payload.idleClaimId === "string" ? payload.idleClaimId.trim() : "";
|
|
2403
|
+
if (idleClaimId && !(await admitIdleStop(slug, payload, idleClaimId))) {
|
|
2404
|
+
json(res, 409, {
|
|
2405
|
+
error: "The idle stop was refused: the session became active or its idle claim is gone.",
|
|
2406
|
+
code: "idle_stop_superseded",
|
|
2407
|
+
});
|
|
2408
|
+
return;
|
|
2409
|
+
}
|
|
2410
|
+
const record = acceptCommand(slug, payload, "stopped");
|
|
2411
|
+
if (idleClaimId) {
|
|
2412
|
+
const key = commandKey(record);
|
|
2413
|
+
idleStopClaims.set(key, idleClaimId);
|
|
2414
|
+
setTimeout(() => idleStopClaims.delete(key), 10 * 60_000).unref();
|
|
2415
|
+
}
|
|
2416
|
+
const scheduledStop = scheduleFencedStop(record);
|
|
2417
|
+
void scheduledStop.completion.catch((error) => {
|
|
2418
|
+
console.error(`[runner] background fenced stop for ${slug} failed:`, error);
|
|
2419
|
+
});
|
|
2420
|
+
const acknowledgement = await scheduledStop.acknowledgement;
|
|
2421
|
+
fenceStore.assertCurrent(record);
|
|
2422
|
+
json(res, 200, {
|
|
2423
|
+
ok: true,
|
|
2424
|
+
runnerId: containerName(slug),
|
|
2425
|
+
callbackExpected: acknowledgement.callbackExpected,
|
|
2426
|
+
...(acknowledgement.startRejectedBeforeWorkspace
|
|
2427
|
+
? { startRejectedBeforeWorkspace: true }
|
|
2428
|
+
: {}),
|
|
2429
|
+
...commandResponse(record),
|
|
2430
|
+
});
|
|
2431
|
+
}
|
|
2432
|
+
catch (error) {
|
|
2433
|
+
requestError(res, error);
|
|
2434
|
+
}
|
|
2435
|
+
return;
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
notFound(res);
|
|
2439
|
+
}
|
|
2440
|
+
const server = http.createServer(async (req, res) => {
|
|
2441
|
+
try {
|
|
2442
|
+
await handleApi(req, res);
|
|
2443
|
+
}
|
|
2444
|
+
catch (err) {
|
|
2445
|
+
console.error("[runner] request failed:", err);
|
|
2446
|
+
json(res, 500, { error: err instanceof Error ? err.message : String(err) });
|
|
2447
|
+
}
|
|
2448
|
+
});
|
|
2449
|
+
server.on("upgrade", (req, socket, head) => {
|
|
2450
|
+
if (!handleTerminalLaneUpgrade(req, socket, head, {
|
|
2451
|
+
config: config.terminalLane ?? null,
|
|
2452
|
+
tickets: terminalLaneTickets,
|
|
2453
|
+
relays: terminalLaneRelays,
|
|
2454
|
+
openUpstream: (slug) => openDockerExecRelay(containerName(slug), {
|
|
2455
|
+
log: (message) => console.log(`[runner] ${slug}: ${message}`),
|
|
2456
|
+
}),
|
|
2457
|
+
log: (message) => console.log(`[runner] ${message}`),
|
|
2458
|
+
})) {
|
|
2459
|
+
socket.destroy();
|
|
2460
|
+
}
|
|
2461
|
+
});
|
|
2462
|
+
return new Promise((resolve, reject) => {
|
|
2463
|
+
server.on("error", (err) => {
|
|
2464
|
+
if (err.code === "EADDRINUSE") {
|
|
2465
|
+
reject(new Error(`Runner port ${port} is already in use — is another runner or ${CLI.command} process running?`));
|
|
2466
|
+
}
|
|
2467
|
+
else {
|
|
2468
|
+
reject(err);
|
|
2469
|
+
}
|
|
2470
|
+
});
|
|
2471
|
+
// Loopback only: the terminal lane must never be reachable from another
|
|
2472
|
+
// machine. Loopback is not enough on its own (containers reach it), so
|
|
2473
|
+
// every request but `/healthz` also needs the API secret.
|
|
2474
|
+
server.listen(port, "127.0.0.1", () => {
|
|
2475
|
+
console.log(`[runner] listening on http://127.0.0.1:${port}`);
|
|
2476
|
+
console.log(`[runner] image=${image}`);
|
|
2477
|
+
try {
|
|
2478
|
+
writeRunnerApiTokenFile(localStateDir, apiToken);
|
|
2479
|
+
}
|
|
2480
|
+
catch (error) {
|
|
2481
|
+
// Only the startup bench reads the file; the Hub session client is
|
|
2482
|
+
// handed the secret in process.
|
|
2483
|
+
console.error("[runner] could not write the API secret file:", error instanceof Error ? error.message : String(error));
|
|
2484
|
+
}
|
|
2485
|
+
console.log("[runner] relying on localhost binding, the runner API secret and this machine's hub credential");
|
|
2486
|
+
recoverAcceptedTerminalCommands();
|
|
2487
|
+
console.log(`[runner] ${stagingDecision.reason}`);
|
|
2488
|
+
reconcileVolumes();
|
|
2489
|
+
const daily = setInterval(reconcileVolumes, RUNNER_VOLUME_RECONCILE_INTERVAL_MS);
|
|
2490
|
+
daily.unref();
|
|
2491
|
+
gitMirrorTimers.push(daily);
|
|
2492
|
+
resolve({
|
|
2493
|
+
server,
|
|
2494
|
+
identity,
|
|
2495
|
+
capabilities: runnerCapabilities,
|
|
2496
|
+
apiToken,
|
|
2497
|
+
countRunningEnvironments: config.countRunningEnvironments ?? (() => countRunningEnvironments(runCapacityDocker)),
|
|
2498
|
+
gadgetJobLoad: () => runnerCapabilities.features?.includes("gadget-job") ? gadgetJobs.load() : null,
|
|
2499
|
+
close: () => {
|
|
2500
|
+
try {
|
|
2501
|
+
removeRunnerApiTokenFile(localStateDir, apiToken);
|
|
2502
|
+
}
|
|
2503
|
+
catch {
|
|
2504
|
+
// A stale file holds a secret no runner accepts.
|
|
2505
|
+
}
|
|
2506
|
+
for (const timer of gitMirrorTimers.splice(0))
|
|
2507
|
+
clearTimeout(timer);
|
|
2508
|
+
for (const { timer } of stageTimers.values())
|
|
2509
|
+
clearTimeout(timer);
|
|
2510
|
+
stageTimers.clear();
|
|
2511
|
+
for (const timer of refillTimers)
|
|
2512
|
+
clearTimeout(timer);
|
|
2513
|
+
refillTimers.clear();
|
|
2514
|
+
void gadgetJobs.close();
|
|
2515
|
+
return closeHttpServer(server);
|
|
2516
|
+
},
|
|
2517
|
+
});
|
|
2518
|
+
});
|
|
2519
|
+
});
|
|
2520
|
+
}
|