overvues 0.0.1 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +58 -2
  2. package/dist/ansi.js +24 -0
  3. package/dist/attach-client.js +586 -0
  4. package/dist/auth-connect-browser.js +112 -0
  5. package/dist/auth-connect.js +306 -0
  6. package/dist/browser-bootstrap.js +190 -0
  7. package/dist/browser-loopback.js +289 -0
  8. package/dist/build-identity.js +6 -0
  9. package/dist/codex-reviewer-apparmor.json +4 -0
  10. package/dist/codex-reviewer-seccomp.json +1238 -0
  11. package/dist/codex-runtime-protocol.js +1 -0
  12. package/dist/codex-subscription.js +53 -0
  13. package/dist/config.js +297 -0
  14. package/dist/connect-comparison-code.js +26 -0
  15. package/dist/development-runner.js +109 -0
  16. package/dist/dns-doctor.js +390 -0
  17. package/dist/docker-names.js +47 -0
  18. package/dist/doctor.js +114 -0
  19. package/dist/gadget-job.js +804 -0
  20. package/dist/health-check.js +112 -0
  21. package/dist/help.js +71 -0
  22. package/dist/host-auth.js +22 -0
  23. package/dist/host-diagnostics.js +20 -0
  24. package/dist/host-runtime-identity.js +39 -0
  25. package/dist/host-runtime-metadata.js +56 -0
  26. package/dist/host-service.js +377 -0
  27. package/dist/host-stack.js +484 -0
  28. package/dist/host-supervisor.js +237 -0
  29. package/dist/host-update.js +387 -0
  30. package/dist/hub-address.js +100 -0
  31. package/dist/hub-client.js +459 -0
  32. package/dist/hub-kind.js +88 -0
  33. package/dist/identity.js +73 -0
  34. package/dist/index.js +185 -0
  35. package/dist/init.js +295 -0
  36. package/dist/isolated-run-cache.js +116 -0
  37. package/dist/local-stack.js +685 -0
  38. package/dist/machine-capacity.js +128 -0
  39. package/dist/managed-runner-image.js +13 -0
  40. package/dist/os-acted-record.js +49 -0
  41. package/dist/os-adapter.js +8190 -0
  42. package/dist/os-command.js +292 -0
  43. package/dist/os-handoff.js +274 -0
  44. package/dist/os-keychain.js +247 -0
  45. package/dist/os-machine.js +818 -0
  46. package/dist/os-owner.js +61 -0
  47. package/dist/os-private-file.js +98 -0
  48. package/dist/os-protocol.js +468 -0
  49. package/dist/os-session-store.js +133 -0
  50. package/dist/os-status.js +102 -0
  51. package/dist/picker.js +863 -0
  52. package/dist/portable-workspace-capability.js +60 -0
  53. package/dist/project-environment-capability.js +62 -0
  54. package/dist/project-environment.js +28 -0
  55. package/dist/project-preparation.js +134 -0
  56. package/dist/readiness.js +28 -0
  57. package/dist/redaction.js +27 -0
  58. package/dist/reviewer-isolation-capability.js +61 -0
  59. package/dist/runner-api-auth.js +80 -0
  60. package/dist/runner-command-fence.js +536 -0
  61. package/dist/runner-inspection.js +57 -0
  62. package/dist/runner-inventory.js +587 -0
  63. package/dist/runner-server.js +2520 -0
  64. package/dist/runner-session.js +470 -0
  65. package/dist/runner-slug-folder.js +39 -0
  66. package/dist/runner-slug.js +40 -0
  67. package/dist/setup.js +139 -0
  68. package/dist/shutdown.js +102 -0
  69. package/dist/splash.js +65 -0
  70. package/dist/staged-workspaces.js +441 -0
  71. package/dist/staging-disk.js +97 -0
  72. package/dist/terminal-lane.js +339 -0
  73. package/dist/terminal-recovery.js +502 -0
  74. package/licenses/capnweb.txt +23 -0
  75. package/licenses/moby-profiles.txt +212 -0
  76. package/licenses/ws.txt +22 -0
  77. package/package.json +28 -7
  78. package/cli.js +0 -2
@@ -0,0 +1,804 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createHash, randomUUID } from "node:crypto";
3
+ import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
4
+ import { join } from "node:path";
5
+ import { dockerLabel, dockerName } from "./docker-names.js";
6
+ import { OS } from "./identity.js";
7
+ /**
8
+ * The build-and-test job, started by the runner (plan 02, "The gates").
9
+ *
10
+ * Given an app's files, the runner starts one container of its own image in
11
+ * the image's `gadget-build` mode (`packages/containers/gadget-build-job.mjs`),
12
+ * hands it the files as a tar on stdin, and reads the result and the built
13
+ * files from a tar on stdout. The job runs the app's own tests, which an agent
14
+ * wrote, so the container is given nothing (plan 05, "Rules for the session"):
15
+ *
16
+ * - no mount of any kind, no published port, no environment file, no Hub
17
+ * address, no credential, no model login and no OS session. The only
18
+ * variable set is the one that selects the job. A request cannot change
19
+ * that: it may carry the app's id, a commit, files and a catalog and
20
+ * nothing else, and any other field is refused, so there is no field a
21
+ * mount, an image or a variable could be named in;
22
+ * - every capability dropped, then only the six the job needs to run the
23
+ * app's commands as an unprivileged user, and no new privileges;
24
+ * - the bridge network, because the app's dependencies come from a
25
+ * registry; a memory limit, a process limit and a time limit.
26
+ *
27
+ * One job per app at a time. A second request for the same app with the same
28
+ * files joins the job that is running and gets its answer; one with different
29
+ * files is refused and is not queued.
30
+ *
31
+ * The runner keeps its own record of every set of built files a job here
32
+ * produced (`gadget-builds.json` in the runner's state folder, readable by
33
+ * this user only). The Hub keeps the files; this record is how the machine
34
+ * later knows that files the Hub hands it for a deploy are ones it built.
35
+ */
36
+ /** The image label that says the image carries the gadget tools and the job. */
37
+ export const GADGET_TOOLS_IMAGE_LABEL = "dev.tiller.gadget-tools";
38
+ /** The image label that says the image's job also maps an OS edit back onto the source. */
39
+ export const GADGET_MAP_BACK_IMAGE_LABEL = "dev.tiller.gadget-map-back";
40
+ export const GADGET_JOB_BOOTSTRAP_MODE = "gadget-build";
41
+ export const GADGET_BUILDS_FILE = "gadget-builds.json";
42
+ export const GADGET_JOB_LIMITS = {
43
+ /** The JSON request, base64 included. */
44
+ maxRequestBytes: 24 * 1024 * 1024,
45
+ maxFiles: 5_000,
46
+ /** The app's files, decoded. */
47
+ maxInputBytes: 12 * 1024 * 1024,
48
+ /** Built files handed back; the OS takes about 4 million characters in one change. */
49
+ maxBuiltBytes: 8 * 1024 * 1024,
50
+ /** What the container may write to stdout before it is stopped. */
51
+ maxOutputBytes: 80 * 1024 * 1024,
52
+ /** From start to answer. Inside the five minutes a local request may wait for its headers. */
53
+ timeoutMs: 270_000,
54
+ memory: "2g",
55
+ pids: 512,
56
+ /** Jobs running at once on this machine, across apps. */
57
+ maxRunning: 2,
58
+ maxRecords: 200,
59
+ };
60
+ /** The capabilities the job needs to run the app's commands as another user; nothing else is kept. */
61
+ export const GADGET_JOB_CAPABILITIES = ["CHOWN", "SETUID", "SETGID", "SETPCAP", "DAC_OVERRIDE", "FOWNER"];
62
+ const REQUEST_FIELDS = new Set(["appId", "commit", "files", "catalog"]);
63
+ const MAP_BACK_FIELDS = new Set(["appId", "commit", "files", "osFiles"]);
64
+ const FILE_FIELDS = new Set(["path", "content", "executable"]);
65
+ const APP_ID = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
66
+ const COMMIT = /^[0-9a-f]{7,64}$/;
67
+ const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/;
68
+ const PROBLEM_KINDS = new Set(["manifest", "layout", "types", "sandbox", "os-limit", "vendor", "catalog", "tests", "install", "tools"]);
69
+ const TEST_STATES = new Set(["passed", "failed", "none", "not-run"]);
70
+ export class GadgetJobRequestError extends Error {
71
+ status;
72
+ code;
73
+ constructor(message, status, code) {
74
+ super(message);
75
+ this.status = status;
76
+ this.code = code;
77
+ this.name = "GadgetJobRequestError";
78
+ }
79
+ }
80
+ const refuse = (message, status = 400, code = "invalid_gadget_job") => {
81
+ throw new GadgetJobRequestError(message, status, code);
82
+ };
83
+ function safeRelativePath(path) {
84
+ if (typeof path !== "string" || !path || path.length > 1000 || path.includes("\\") || path.includes("\0"))
85
+ return null;
86
+ const parts = path.split("/");
87
+ if (parts.some((part) => !part || part === "." || part === ".."))
88
+ return null;
89
+ return path;
90
+ }
91
+ function isRecord(value) {
92
+ return typeof value === "object" && value !== null && !Array.isArray(value);
93
+ }
94
+ /** Decodes a request's list of files, counting their bytes against the job's limit. */
95
+ function decodeFiles(listed, total, what = "The app") {
96
+ const files = new Map();
97
+ const executable = new Set();
98
+ for (const entry of listed) {
99
+ if (!isRecord(entry))
100
+ return refuse("A file is not an object.");
101
+ for (const key of Object.keys(entry))
102
+ if (!FILE_FIELDS.has(key))
103
+ refuse(`A file cannot carry "${key.slice(0, 40)}".`);
104
+ const path = safeRelativePath(entry.path);
105
+ if (!path)
106
+ return refuse("A file's path is not a path inside the app.");
107
+ if (files.has(path))
108
+ return refuse(`${path} is listed twice.`);
109
+ if (typeof entry.content !== "string" || entry.content.length % 4 !== 0 || !BASE64.test(entry.content))
110
+ return refuse(`${path} is not base64.`);
111
+ if (entry.executable !== undefined && typeof entry.executable !== "boolean")
112
+ return refuse(`${path} has an executable mark that is not true or false.`);
113
+ const content = Buffer.from(entry.content, "base64");
114
+ total.bytes += content.length;
115
+ if (total.bytes > GADGET_JOB_LIMITS.maxInputBytes)
116
+ return refuse(`${what} is larger than a build-and-test job accepts (${GADGET_JOB_LIMITS.maxInputBytes / (1024 * 1024)} MB).`, 413, "gadget_job_too_large");
117
+ files.set(path, content);
118
+ if (entry.executable === true)
119
+ executable.add(path);
120
+ }
121
+ return { files, executable };
122
+ }
123
+ /**
124
+ * Checks a request and decodes its files. Only the four known fields are
125
+ * read; any other is refused, so a request cannot name a mount, a port, an
126
+ * image, a network or a variable for the container.
127
+ */
128
+ export function parseGadgetJobRequest(body) {
129
+ if (!isRecord(body))
130
+ return refuse("Expected a JSON object.");
131
+ for (const key of Object.keys(body)) {
132
+ if (!REQUEST_FIELDS.has(key))
133
+ refuse(`A build-and-test request cannot carry "${key.slice(0, 40)}". It takes an app's id, a commit, files and a catalog; the container's settings are the runner's alone.`);
134
+ }
135
+ const appId = body.appId;
136
+ if (typeof appId !== "string" || !APP_ID.test(appId) || appId.includes(".."))
137
+ return refuse("The app's id is not a plain name.");
138
+ const commit = body.commit === undefined || body.commit === null ? null : body.commit;
139
+ if (commit !== null && (typeof commit !== "string" || !COMMIT.test(commit)))
140
+ return refuse("The commit is not a commit id.");
141
+ if (!Array.isArray(body.files) || !body.files.length)
142
+ return refuse("The request has no files.");
143
+ if (body.files.length > GADGET_JOB_LIMITS.maxFiles)
144
+ return refuse("The app has more files than a build-and-test job accepts.", 413, "gadget_job_too_large");
145
+ const { files, executable } = decodeFiles(body.files, { bytes: 0 });
146
+ let catalog = null;
147
+ if (body.catalog !== undefined && body.catalog !== null) {
148
+ if (typeof body.catalog !== "string" || body.catalog.length > 1024 * 1024)
149
+ return refuse("The catalog is not JSON text.");
150
+ try {
151
+ JSON.parse(body.catalog);
152
+ }
153
+ catch {
154
+ return refuse("The catalog is not JSON text.");
155
+ }
156
+ catalog = body.catalog;
157
+ }
158
+ const hash = createHash("sha256");
159
+ for (const path of [...files.keys()].sort()) {
160
+ const content = files.get(path);
161
+ hash.update(`${Buffer.byteLength(path)}\n${path}${executable.has(path) ? "x" : "-"}${content.length}\n`).update(content);
162
+ }
163
+ hash.update(catalog === null ? "no catalog" : `catalog ${Buffer.byteLength(catalog)}\n${catalog}`);
164
+ return { appId, commit, files, executable, catalog, inputDigest: hash.digest("hex") };
165
+ }
166
+ // --- tar: plain ustar, the form the job reads and writes (gadget-build-job.mjs) ----------------
167
+ /** A tar of regular files. */
168
+ export function packTar(files, executable = new Set()) {
169
+ const blocks = [];
170
+ for (const [path, body] of files) {
171
+ const content = typeof body === "string" ? Buffer.from(body, "utf8") : body;
172
+ let name = path;
173
+ let prefix = "";
174
+ if (Buffer.byteLength(name) > 100) {
175
+ const cut = path.lastIndexOf("/", 155);
176
+ prefix = path.slice(0, cut);
177
+ name = path.slice(cut + 1);
178
+ if (cut <= 0 || Buffer.byteLength(prefix) > 155 || Buffer.byteLength(name) > 100) {
179
+ refuse(`${path.slice(0, 80)}… has a path too long to hand to the job.`);
180
+ }
181
+ }
182
+ const header = Buffer.alloc(512);
183
+ header.write(name, 0, 100, "utf8");
184
+ header.write(executable.has(path) ? "0000755\0" : "0000644\0", 100, 8, "ascii");
185
+ header.write("0000000\0", 108, 8, "ascii");
186
+ header.write("0000000\0", 116, 8, "ascii");
187
+ header.write(content.length.toString(8).padStart(11, "0") + "\0", 124, 12, "ascii");
188
+ header.write("00000000000\0", 136, 12, "ascii");
189
+ header.write(" ", 148, 8, "ascii");
190
+ header.write("0", 156, 1, "ascii");
191
+ header.write("ustar\0", 257, 6, "ascii");
192
+ header.write("00", 263, 2, "ascii");
193
+ header.write(prefix, 345, 155, "utf8");
194
+ let sum = 0;
195
+ for (const byte of header)
196
+ sum += byte;
197
+ header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, 8, "ascii");
198
+ blocks.push(header, content, Buffer.alloc((512 - (content.length % 512)) % 512));
199
+ }
200
+ blocks.push(Buffer.alloc(1024));
201
+ return Buffer.concat(blocks);
202
+ }
203
+ /** The regular files of a plain ustar archive, by path. Anything else in it is an error. */
204
+ export function unpackTar(buffer) {
205
+ const text = (start, length) => buffer.subarray(start, start + length).toString("utf8").replace(/\0.*$/s, "");
206
+ const files = new Map();
207
+ let offset = 0;
208
+ while (offset + 512 <= buffer.length) {
209
+ if (buffer.subarray(offset, offset + 512).every((byte) => byte === 0))
210
+ break;
211
+ const rawName = text(offset, 100);
212
+ const prefix = text(offset + 345, 155);
213
+ const size = Number.parseInt(text(offset + 124, 12).trim() || "0", 8);
214
+ const typeFlag = text(offset + 156, 1);
215
+ offset += 512;
216
+ if (!Number.isFinite(size) || size < 0 || offset + size > buffer.length)
217
+ throw new Error("the job's output is cut short");
218
+ if (typeFlag !== "0" && typeFlag !== "")
219
+ throw new Error("the job's output holds something that is not a plain file");
220
+ const name = prefix ? `${prefix}/${rawName}` : rawName;
221
+ if (files.has(name))
222
+ throw new Error("the job's output lists a file twice");
223
+ files.set(name, Buffer.from(buffer.subarray(offset, offset + size)));
224
+ offset += Math.ceil(size / 512) * 512;
225
+ }
226
+ return files;
227
+ }
228
+ /** The tar the job reads: `app/<path>` for each file and, when given, `catalog.json`. */
229
+ export function buildGadgetJobInput(request) {
230
+ const entries = [...request.files].map(([path, content]) => [`app/${path}`, content]);
231
+ if (request.catalog !== null)
232
+ entries.push(["catalog.json", request.catalog]);
233
+ return packTar(entries, new Set([...request.executable].map((path) => `app/${path}`)));
234
+ }
235
+ // --- the container ------------------------------------------------------------------------------
236
+ /** The label every job container carries; its value is the app's id. */
237
+ export function gadgetJobLabel() {
238
+ return dockerLabel("gadget-job");
239
+ }
240
+ /** One name per app, so Docker itself refuses a second job for the same app. */
241
+ export function gadgetJobContainerName(appId) {
242
+ return dockerName(`gadget-job-${createHash("sha256").update(appId).digest("hex").slice(0, 20)}`);
243
+ }
244
+ /**
245
+ * `docker run` for one job. Nothing in it comes from the request but the
246
+ * app's id, as a label; the name is a hash of that id. There is no `-v`,
247
+ * `--mount`, `--volumes-from`, `-p`, `-P`, `--env-file`, `--add-host` or
248
+ * `--privileged`, and the one variable is the mode that selects the job.
249
+ */
250
+ export function buildGadgetJobRunArgs(appId, image) {
251
+ return [
252
+ "run", "--rm", "-i", "--pull=never",
253
+ "--name", gadgetJobContainerName(appId),
254
+ "--label", `${gadgetJobLabel()}=${appId}`,
255
+ "--network", "bridge",
256
+ "--cap-drop", "ALL",
257
+ ...GADGET_JOB_CAPABILITIES.flatMap((capability) => ["--cap-add", capability]),
258
+ "--security-opt", "no-new-privileges",
259
+ "--memory", GADGET_JOB_LIMITS.memory,
260
+ "--memory-swap", GADGET_JOB_LIMITS.memory,
261
+ "--pids-limit", String(GADGET_JOB_LIMITS.pids),
262
+ "-e", `TILLER_BOOTSTRAP_MODE=${GADGET_JOB_BOOTSTRAP_MODE}`,
263
+ image,
264
+ ];
265
+ }
266
+ /** Starts the container, writes the tar to its stdin and collects its stdout. */
267
+ export const runGadgetJobContainer = (args, input, timeoutMs, kill) => new Promise((resolvePromise, reject) => {
268
+ const child = spawn("docker", args, { stdio: ["pipe", "pipe", "pipe"] });
269
+ const chunks = [];
270
+ let bytes = 0;
271
+ let stderrTail = "";
272
+ let timedOut = false;
273
+ let tooLarge = false;
274
+ const timer = setTimeout(() => {
275
+ timedOut = true;
276
+ kill();
277
+ }, timeoutMs);
278
+ child.stdout.on("data", (chunk) => {
279
+ bytes += chunk.length;
280
+ if (bytes > GADGET_JOB_LIMITS.maxOutputBytes) {
281
+ if (!tooLarge)
282
+ kill();
283
+ tooLarge = true;
284
+ return;
285
+ }
286
+ chunks.push(chunk);
287
+ });
288
+ child.stderr.on("data", (chunk) => {
289
+ stderrTail = (stderrTail + chunk.toString("utf8")).slice(-4_000);
290
+ });
291
+ child.on("error", (error) => {
292
+ clearTimeout(timer);
293
+ reject(error);
294
+ });
295
+ child.on("close", (status) => {
296
+ clearTimeout(timer);
297
+ resolvePromise({ status, stdout: Buffer.concat(chunks), stderrTail, timedOut, tooLarge });
298
+ });
299
+ // A container that never reads its input (it failed to start) must not take the runner down.
300
+ child.stdin.on("error", () => { });
301
+ child.stdin.end(input);
302
+ });
303
+ // --- the job's answer ---------------------------------------------------------------------------
304
+ const text = (value, max) => String(typeof value === "string" ? value : "").slice(0, max);
305
+ const wholeMs = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.round(value) : undefined);
306
+ /** A result for a job that gave no usable answer. Nothing is known about the app. */
307
+ export function gadgetJobError(message) {
308
+ return {
309
+ result: 1,
310
+ status: "error",
311
+ problems: [{ kind: "tools", file: null, message: message.slice(0, 4_000), osWouldRefuse: false }],
312
+ warnings: [],
313
+ tests: "not-run",
314
+ built: null,
315
+ blocksShipAlways: true,
316
+ tools: { version: "unknown", node: "unknown" },
317
+ timings: { totalMs: 0 },
318
+ separated: false,
319
+ logs: {},
320
+ };
321
+ }
322
+ /**
323
+ * Reads the job's output tar. The result is rebuilt field by field, so only
324
+ * the known shape leaves the runner, and the built files are checked against
325
+ * the list the job made of them before any code of the app ran: every listed
326
+ * file is there with that size and SHA-256, and nothing else is.
327
+ */
328
+ export function readGadgetJobOutput(tar) {
329
+ const entries = unpackTar(tar);
330
+ const raw = JSON.parse((entries.get("result.json") ?? Buffer.from("null")).toString("utf8"));
331
+ if (!isRecord(raw) || raw.result !== 1 || (raw.status !== "conforms" && raw.status !== "needs-fix" && raw.status !== "error") || !Array.isArray(raw.problems)) {
332
+ throw new Error("the job's output has no result");
333
+ }
334
+ const problems = raw.problems.slice(0, 500).filter(isRecord).map((p) => ({
335
+ kind: PROBLEM_KINDS.has(p.kind) ? p.kind : "tools",
336
+ file: typeof p.file === "string" && p.file ? p.file.slice(0, 500) : null,
337
+ ...(Number.isInteger(p.line) ? { line: p.line } : {}),
338
+ message: text(p.message, 4_000),
339
+ osWouldRefuse: p.osWouldRefuse === true,
340
+ }));
341
+ const built = new Map();
342
+ let listed = null;
343
+ if (raw.built !== null && raw.built !== undefined) {
344
+ if (!isRecord(raw.built) || typeof raw.built.fingerprint !== "string" || !/^sha256:[0-9a-f]{64}$/.test(raw.built.fingerprint) || !Array.isArray(raw.built.files)) {
345
+ throw new Error("the job's list of built files is not usable");
346
+ }
347
+ const files = [];
348
+ for (const file of raw.built.files) {
349
+ const path = isRecord(file) ? safeRelativePath(file.path) : null;
350
+ if (!isRecord(file) || !path || built.has(path))
351
+ throw new Error("the job's list of built files is not usable");
352
+ const content = entries.get(`built/${path}`);
353
+ if (!content || content.length !== file.bytes || createHash("sha256").update(content).digest("hex") !== file.sha256) {
354
+ throw new Error(`the built file ${path} is not the one the job listed`);
355
+ }
356
+ built.set(path, content);
357
+ files.push({ path, bytes: content.length, sha256: file.sha256 });
358
+ }
359
+ listed = { fingerprint: raw.built.fingerprint, files };
360
+ }
361
+ const extra = [...entries.keys()].filter((name) => name !== "result.json" && !(name.startsWith("built/") && built.has(name.slice(6))));
362
+ if (extra.length)
363
+ throw new Error("the job's output holds files it did not list");
364
+ const tools = isRecord(raw.tools) ? raw.tools : {};
365
+ const timings = isRecord(raw.timings) ? raw.timings : {};
366
+ const logs = isRecord(raw.logs) ? raw.logs : {};
367
+ const status = raw.status;
368
+ if (status === "conforms" && (problems.length || !listed))
369
+ throw new Error("the job says the app conforms and lists problems or no built files");
370
+ const result = {
371
+ result: 1,
372
+ status,
373
+ problems,
374
+ warnings: (Array.isArray(raw.warnings) ? raw.warnings : []).slice(0, 100).map((warning) => text(warning, 1_000)),
375
+ tests: TEST_STATES.has(raw.tests) ? raw.tests : "not-run",
376
+ built: listed,
377
+ // Worked out again here from what was checked, not taken from the job's word.
378
+ blocksShipAlways: listed === null || problems.some((p) => p.osWouldRefuse),
379
+ tools: { version: text(tools.version, 100) || "unknown", node: text(tools.node, 100) || "unknown" },
380
+ timings: {
381
+ ...(wholeMs(timings.installMs) !== undefined ? { installMs: wholeMs(timings.installMs) } : {}),
382
+ ...(wholeMs(timings.buildMs) !== undefined ? { buildMs: wholeMs(timings.buildMs) } : {}),
383
+ ...(wholeMs(timings.testMs) !== undefined ? { testMs: wholeMs(timings.testMs) } : {}),
384
+ totalMs: wholeMs(timings.totalMs) ?? 0,
385
+ },
386
+ separated: raw.separated === true,
387
+ logs: {
388
+ ...(typeof logs.install === "string" ? { install: logs.install.slice(-8_000) } : {}),
389
+ ...(typeof logs.build === "string" ? { build: logs.build.slice(-8_000) } : {}),
390
+ ...(typeof logs.test === "string" ? { test: logs.test.slice(-8_000) } : {}),
391
+ },
392
+ };
393
+ return { result, built };
394
+ }
395
+ /**
396
+ * Checks a mapping-back request and decodes its files. It may carry the app's id, a commit, the
397
+ * app's source and the OS's files and nothing else, for the reason `parseGadgetJobRequest` gives.
398
+ * The two sets of files together stay inside the job's one limit.
399
+ */
400
+ export function parseGadgetMapBackRequest(body) {
401
+ if (!isRecord(body))
402
+ return refuse("Expected a JSON object.");
403
+ for (const key of Object.keys(body)) {
404
+ if (!MAP_BACK_FIELDS.has(key))
405
+ refuse(`A mapping-back request cannot carry "${key.slice(0, 40)}". It takes an app's id, a commit, the app's files and ${OS.the}'s files; the container's settings are the runner's alone.`);
406
+ }
407
+ const appId = body.appId;
408
+ if (typeof appId !== "string" || !APP_ID.test(appId) || appId.includes(".."))
409
+ return refuse("The app's id is not a plain name.");
410
+ const commit = body.commit === undefined || body.commit === null ? null : body.commit;
411
+ if (commit !== null && (typeof commit !== "string" || !COMMIT.test(commit)))
412
+ return refuse("The commit is not a commit id.");
413
+ if (!Array.isArray(body.files) || !body.files.length)
414
+ return refuse("The request has no files.");
415
+ if (!Array.isArray(body.osFiles) || !body.osFiles.length)
416
+ return refuse(`The request has none of ${OS.the}'s files.`);
417
+ if (body.files.length + body.osFiles.length > GADGET_JOB_LIMITS.maxFiles)
418
+ return refuse("The app has more files than a build-and-test job accepts.", 413, "gadget_job_too_large");
419
+ const total = { bytes: 0 };
420
+ const { files } = decodeFiles(body.files, total);
421
+ const { files: osFiles, executable } = decodeFiles(body.osFiles, total, `The app with ${OS.the}'s files`);
422
+ if (executable.size)
423
+ return refuse(`One of ${OS.the}'s files is marked as a program; ${OS.the} holds text.`);
424
+ const hash = createHash("sha256").update("map-back\n");
425
+ for (const [label, set] of [["app", files], ["os", osFiles]]) {
426
+ for (const path of [...set.keys()].sort()) {
427
+ const content = set.get(path);
428
+ hash.update(`${label} ${Buffer.byteLength(path)}\n${path}${content.length}\n`).update(content);
429
+ }
430
+ }
431
+ return { appId, commit, files, osFiles, inputDigest: hash.digest("hex") };
432
+ }
433
+ /** The tar the job reads for mapping back: `map-back.json`, `app/<path>` and `os/<path>`. */
434
+ export function buildGadgetMapBackInput(request) {
435
+ return packTar([
436
+ ["map-back.json", JSON.stringify({ mapBack: 1 })],
437
+ ...[...request.files].map(([path, content]) => [`app/${path}`, content]),
438
+ ...[...request.osFiles].map(([path, content]) => [`os/${path}`, content]),
439
+ ]);
440
+ }
441
+ /** A mapping-back result for a job that gave no usable answer. Nothing is known. */
442
+ export function gadgetMapBackError(message) {
443
+ return { mapBack: 1, status: "error", changes: [], unmappable: [], notPulled: [], notes: [], error: message.slice(0, 4_000), tools: { version: "unknown", node: "unknown" }, timings: { totalMs: 0 } };
444
+ }
445
+ /**
446
+ * Reads the mapping-back job's output tar. The result is rebuilt field by field, and the files
447
+ * handed back must be exactly the ones the account says are added or changed.
448
+ */
449
+ export function readGadgetMapBackOutput(tar) {
450
+ const entries = unpackTar(tar);
451
+ const raw = JSON.parse((entries.get("result.json") ?? Buffer.from("null")).toString("utf8"));
452
+ if (!isRecord(raw) || raw.mapBack !== 1 || (raw.status !== "mapped" && raw.status !== "unmappable" && raw.status !== "error")) {
453
+ // An image without this mode answers with a build-and-test result that says why it refused the input.
454
+ if (isRecord(raw) && raw.result === 1)
455
+ throw new Error(`this machine's image does not map ${OS.noun} edits back`);
456
+ throw new Error("the job's output has no mapping-back result");
457
+ }
458
+ const status = raw.status;
459
+ const reasons = (value) => (Array.isArray(value) ? value : []).slice(0, 2_000).filter(isRecord)
460
+ .map((entry) => ({ file: text(entry.file, 500), reason: text(entry.reason, 4_000) }));
461
+ const changes = [];
462
+ const source = new Map();
463
+ for (const entry of Array.isArray(raw.changes) ? raw.changes.slice(0, 2_000) : []) {
464
+ const path = isRecord(entry) ? safeRelativePath(entry.path) : null;
465
+ if (!isRecord(entry) || !path || (entry.change !== "added" && entry.change !== "changed" && entry.change !== "removed") || changes.some((change) => change.path === path)) {
466
+ throw new Error("the job's account of what changed is not usable");
467
+ }
468
+ changes.push({ path, change: entry.change, from: text(entry.from, 500) });
469
+ if (status !== "mapped" || entry.change === "removed")
470
+ continue;
471
+ const content = entries.get(`source/${path}`);
472
+ if (!content)
473
+ throw new Error(`the job says ${path} changed and did not hand it back`);
474
+ source.set(path, content);
475
+ }
476
+ const extra = [...entries.keys()].filter((name) => name !== "result.json" && !(name.startsWith("source/") && source.has(name.slice(7))));
477
+ if (extra.length)
478
+ throw new Error("the job's output holds files it did not list");
479
+ const unmappable = reasons(raw.unmappable);
480
+ if (status === "unmappable" && !unmappable.length)
481
+ throw new Error("the job says the edit cannot be mapped back and gives no reason");
482
+ if (status === "mapped" && unmappable.length)
483
+ throw new Error("the job says the edit was mapped back and lists files that cannot be");
484
+ const tools = isRecord(raw.tools) ? raw.tools : {};
485
+ const timings = isRecord(raw.timings) ? raw.timings : {};
486
+ return {
487
+ result: {
488
+ mapBack: 1,
489
+ status,
490
+ changes,
491
+ unmappable,
492
+ notPulled: reasons(raw.notPulled),
493
+ notes: (Array.isArray(raw.notes) ? raw.notes : []).slice(0, 200).map((note) => text(note, 1_000)),
494
+ ...(status === "error" ? { error: text(raw.error, 4_000) || "the job gave no reason" } : {}),
495
+ tools: { version: text(tools.version, 100) || "unknown", node: text(tools.node, 100) || "unknown" },
496
+ timings: { totalMs: wholeMs(timings.totalMs) ?? 0 },
497
+ },
498
+ source,
499
+ };
500
+ }
501
+ // --- this machine's record of what it built --------------------------------------------------------
502
+ function buildsPath(localStateDir) {
503
+ return join(localStateDir, GADGET_BUILDS_FILE);
504
+ }
505
+ /** Every build this machine recorded, oldest first. An unreadable file reads as none. */
506
+ export function readGadgetBuildRecords(localStateDir) {
507
+ try {
508
+ const stored = JSON.parse(readFileSync(buildsPath(localStateDir), "utf8"));
509
+ if (!isRecord(stored) || stored.version !== 1 || !Array.isArray(stored.builds))
510
+ return [];
511
+ return stored.builds.filter((entry) => isRecord(entry) && typeof entry.appId === "string" && typeof entry.fingerprint === "string" && typeof entry.filesDigest === "string");
512
+ }
513
+ catch {
514
+ return [];
515
+ }
516
+ }
517
+ /**
518
+ * The record of a build of this app with this fingerprint, if a job on this
519
+ * machine produced one. For the deploy: files the Hub hands over are only
520
+ * sent to the OS when their fingerprint, worked out again here, is one this
521
+ * machine recorded for the app.
522
+ */
523
+ export function findGadgetBuildRecord(localStateDir, appId, fingerprint) {
524
+ return readGadgetBuildRecords(localStateDir).reverse().find((entry) => entry.appId === appId && entry.fingerprint === fingerprint) ?? null;
525
+ }
526
+ function recordGadgetBuild(localStateDir, record) {
527
+ mkdirSync(localStateDir, { recursive: true, mode: 0o700 });
528
+ const builds = readGadgetBuildRecords(localStateDir)
529
+ .filter((entry) => !(entry.appId === record.appId && entry.fingerprint === record.fingerprint && entry.inputDigest === record.inputDigest));
530
+ builds.push(record);
531
+ const path = buildsPath(localStateDir);
532
+ const temporary = `${path}.${process.pid}.tmp`;
533
+ try {
534
+ writeFileSync(temporary, `${JSON.stringify({ version: 1, builds: builds.slice(-GADGET_JOB_LIMITS.maxRecords) }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
535
+ chmodSync(temporary, 0o600);
536
+ renameSync(temporary, path);
537
+ }
538
+ catch (error) {
539
+ rmSync(temporary, { force: true });
540
+ throw error;
541
+ }
542
+ }
543
+ /** A digest of the built files as the runner itself read them. */
544
+ function builtFilesDigest(built) {
545
+ const hash = createHash("sha256");
546
+ for (const path of [...built.keys()].sort()) {
547
+ const content = built.get(path);
548
+ hash.update(`${Buffer.byteLength(path)}\n${path}${content.length}\n${createHash("sha256").update(content).digest("hex")}\n`);
549
+ }
550
+ return hash.digest("hex");
551
+ }
552
+ /**
553
+ * The OS adapter's fingerprint of a set of built files as text (`fingerprintFiles` in
554
+ * `tiller-os-adapter`), by the same written rule: for each file in order of path, the path's
555
+ * length in UTF-8 bytes, a newline, the path, the text's length in UTF-8 bytes, a newline, the
556
+ * text. Written out here so the runner's local API does not load the adapter, which is the
557
+ * OS side's; a test holds the two to the same answers.
558
+ */
559
+ export function builtFilesFingerprint(files) {
560
+ const hash = createHash("sha256");
561
+ for (const path of Object.keys(files).sort()) {
562
+ const text = files[path];
563
+ hash.update(`${Buffer.byteLength(path, "utf8")}\n`).update(path, "utf8").update(`${Buffer.byteLength(text, "utf8")}\n`).update(text, "utf8");
564
+ }
565
+ return `sha256:${hash.digest("hex")}`;
566
+ }
567
+ export class GadgetJobs {
568
+ options;
569
+ /** One job per app at a time, of either mode: both run in the container named for the app. */
570
+ running = new Map();
571
+ leftoversRemoved = null;
572
+ constructor(options) {
573
+ this.options = options;
574
+ }
575
+ /** Jobs running now (of either mode) and how many this machine allows, for the capacity report. */
576
+ load() {
577
+ return { running: this.running.size, limit: GADGET_JOB_LIMITS.maxRunning };
578
+ }
579
+ /**
580
+ * Runs the job for one app. Throws `GadgetJobRequestError` when the request
581
+ * is refused (another job for the app is running with different files, or
582
+ * the machine is running as many jobs as it allows). Every other failure is
583
+ * an answer whose result has the status `error` and a plain reason.
584
+ */
585
+ async run(request) {
586
+ const current = this.running.get(request.appId);
587
+ if (current) {
588
+ if (current.inputDigest !== request.inputDigest) {
589
+ throw new GadgetJobRequestError("A build and test of this app is already running on this machine, for other files. Try again when it has finished.", 409, "gadget_job_busy");
590
+ }
591
+ return { ...(await current.work), joined: true };
592
+ }
593
+ if (this.running.size >= GADGET_JOB_LIMITS.maxRunning) {
594
+ throw new GadgetJobRequestError("This machine is already building and testing other apps. Try again when one has finished.", 409, "gadget_jobs_full");
595
+ }
596
+ const work = this.runOne(request).finally(() => {
597
+ if (this.running.get(request.appId)?.work === work)
598
+ this.running.delete(request.appId);
599
+ });
600
+ this.running.set(request.appId, { inputDigest: request.inputDigest, work });
601
+ return work;
602
+ }
603
+ /**
604
+ * Maps an OS edit back onto one app's source: the job's second mode, in the same container,
605
+ * under the same limits and the same one-job-per-app rule. Nothing is recorded: no files are
606
+ * built, so there is nothing this machine could later be asked to deploy.
607
+ */
608
+ async mapBack(request) {
609
+ const current = this.running.get(request.appId);
610
+ if (current) {
611
+ if (current.inputDigest !== request.inputDigest) {
612
+ throw new GadgetJobRequestError("A build and test of this app is already running on this machine, for other files. Try again when it has finished.", 409, "gadget_job_busy");
613
+ }
614
+ return { ...(await current.work), joined: true };
615
+ }
616
+ if (this.running.size >= GADGET_JOB_LIMITS.maxRunning) {
617
+ throw new GadgetJobRequestError("This machine is already building and testing other apps. Try again when one has finished.", 409, "gadget_jobs_full");
618
+ }
619
+ const work = this.mapBackOne(request).finally(() => {
620
+ if (this.running.get(request.appId)?.work === work)
621
+ this.running.delete(request.appId);
622
+ });
623
+ this.running.set(request.appId, { inputDigest: request.inputDigest, work });
624
+ return work;
625
+ }
626
+ async mapBackOne(request) {
627
+ const jobId = randomUUID();
628
+ const base = { ok: true, jobId, appId: request.appId, commit: request.commit, inputDigest: request.inputDigest, joined: false };
629
+ const failed = (message) => ({ ...base, result: gadgetMapBackError(message), source: {} });
630
+ const name = gadgetJobContainerName(request.appId);
631
+ const timeoutMs = this.options.timeoutMs ?? GADGET_JOB_LIMITS.timeoutMs;
632
+ let outcome;
633
+ try {
634
+ const input = buildGadgetMapBackInput(request);
635
+ await this.removeLeftovers();
636
+ outcome = await (this.options.runContainer ?? runGadgetJobContainer)(buildGadgetJobRunArgs(request.appId, this.options.image), input, timeoutMs, () => void this.options.runDocker(["kill", name]).catch(() => undefined));
637
+ }
638
+ catch (error) {
639
+ if (error instanceof GadgetJobRequestError)
640
+ throw error;
641
+ await this.remove(name);
642
+ return failed(`The job that maps an ${OS.noun} edit back could not start on this machine: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`);
643
+ }
644
+ if (outcome.timedOut || outcome.tooLarge)
645
+ await this.remove(name);
646
+ if (outcome.timedOut)
647
+ return failed(`Mapping the ${OS.noun} edit back did not finish in ${Math.round(timeoutMs / 1000)} seconds, so it was stopped.`);
648
+ if (outcome.tooLarge)
649
+ return failed(`The job that maps an ${OS.noun} edit back handed back more than the runner accepts, so it was stopped.`);
650
+ let read;
651
+ try {
652
+ read = readGadgetMapBackOutput(outcome.stdout);
653
+ }
654
+ catch (error) {
655
+ const detail = outcome.stdout.length ? (error instanceof Error ? error.message : String(error)) : `the container ended with status ${outcome.status ?? "unknown"} and no output`;
656
+ this.options.log?.(`[runner] mapping-back job ${jobId} gave no usable answer: ${detail}`);
657
+ return failed(`The job that maps an ${OS.noun} edit back gave no usable answer (${detail.slice(0, 300)}). Update the machine.`);
658
+ }
659
+ const { result } = read;
660
+ this.options.log?.(`[runner] mapping-back job ${jobId} for ${request.appId}: ${result.status}, ${result.changes.length} source file(s) change, ${result.unmappable.length} cannot be mapped`);
661
+ const bytes = [...read.source.values()].reduce((total, content) => total + content.length, 0);
662
+ if (bytes > GADGET_JOB_LIMITS.maxBuiltBytes)
663
+ return failed(`The changed source files are ${Math.ceil(bytes / (1024 * 1024))} MB, more than can be handed back.`);
664
+ const source = Object.create(null);
665
+ for (const [path, content] of read.source) {
666
+ const asText = content.toString("utf8");
667
+ // The OS holds text, so every file a pull writes is text.
668
+ if (!Buffer.from(asText, "utf8").equals(content))
669
+ return failed(`${path} came back from the job as something other than text.`);
670
+ source[path] = asText;
671
+ }
672
+ return { ...base, result, source };
673
+ }
674
+ /** Stops every job container this runner started. */
675
+ async close() {
676
+ await Promise.all([...this.running.keys()].map((appId) => this.remove(gadgetJobContainerName(appId))));
677
+ }
678
+ async remove(name) {
679
+ await this.options.runDocker(["rm", "-f", name]).catch(() => undefined);
680
+ }
681
+ /**
682
+ * Job containers an earlier runner left behind (it stopped while one ran),
683
+ * found by their label and removed, once, before this runner's first job.
684
+ */
685
+ removeLeftovers() {
686
+ this.leftoversRemoved ??= (async () => {
687
+ const ids = (await this.options.runDocker(["ps", "-aq", "--filter", `label=${gadgetJobLabel()}`]).catch(() => ""))
688
+ .split("\n").map((id) => id.trim()).filter((id) => /^[0-9a-f]{12,64}$/.test(id));
689
+ const mine = new Set([...this.running.keys()].map(gadgetJobContainerName));
690
+ for (const id of ids) {
691
+ const name = (await this.options.runDocker(["inspect", "--type", "container", "-f", "{{.Name}}", id]).catch(() => "")).replace(/^\//, "");
692
+ if (name && !mine.has(name))
693
+ await this.remove(id);
694
+ }
695
+ })();
696
+ return this.leftoversRemoved;
697
+ }
698
+ async runOne(request) {
699
+ const jobId = randomUUID();
700
+ const base = { ok: true, jobId, appId: request.appId, commit: request.commit, inputDigest: request.inputDigest, joined: false };
701
+ const failed = (message) => ({ ...base, result: gadgetJobError(message), built: null, record: null });
702
+ const name = gadgetJobContainerName(request.appId);
703
+ const timeoutMs = this.options.timeoutMs ?? GADGET_JOB_LIMITS.timeoutMs;
704
+ const startedAt = performance.now();
705
+ let outcome;
706
+ try {
707
+ const input = buildGadgetJobInput(request);
708
+ await this.removeLeftovers();
709
+ outcome = await (this.options.runContainer ?? runGadgetJobContainer)(buildGadgetJobRunArgs(request.appId, this.options.image), input, timeoutMs, () => void this.options.runDocker(["kill", name]).catch(() => undefined));
710
+ }
711
+ catch (error) {
712
+ if (error instanceof GadgetJobRequestError)
713
+ throw error;
714
+ await this.remove(name);
715
+ return failed(`The build-and-test job could not start on this machine: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`);
716
+ }
717
+ // `--rm` removes a container that ended; one that was killed or never ended is removed here.
718
+ if (outcome.timedOut || outcome.tooLarge)
719
+ await this.remove(name);
720
+ const seconds = Math.round((performance.now() - startedAt) / 1000);
721
+ if (outcome.timedOut)
722
+ return failed(`Build and tests did not finish in ${Math.round(timeoutMs / 1000)} seconds, so they were stopped.`);
723
+ if (outcome.tooLarge)
724
+ return failed("The build-and-test job handed back more than the runner accepts, so it was stopped.");
725
+ let read;
726
+ try {
727
+ read = readGadgetJobOutput(outcome.stdout);
728
+ }
729
+ catch (error) {
730
+ const detail = outcome.stdout.length ? (error instanceof Error ? error.message : String(error)) : `the container ended with status ${outcome.status ?? "unknown"} and no output`;
731
+ this.options.log?.(`[runner] build-and-test job ${jobId} gave no usable answer: ${detail}`);
732
+ return failed(`The build-and-test job gave no usable answer (${detail.slice(0, 300)}). The runner's image may not carry the job; update the machine.`);
733
+ }
734
+ const { result, built } = read;
735
+ this.options.log?.(`[runner] build-and-test job ${jobId} for ${request.appId}: ${result.status}, ${result.problems.length} problem(s), tests ${result.tests}, ${built.size} built file(s), ${seconds} s`);
736
+ if (!result.built || result.status === "error")
737
+ return { ...base, result, built: null, record: null };
738
+ const bytes = [...built.values()].reduce((total, content) => total + content.length, 0);
739
+ if (bytes > GADGET_JOB_LIMITS.maxBuiltBytes) {
740
+ return { ...base, result, built: null, record: null, builtOmitted: `The built files are ${Math.ceil(bytes / (1024 * 1024))} MB, more than can be handed back (${GADGET_JOB_LIMITS.maxBuiltBytes / (1024 * 1024)} MB).` };
741
+ }
742
+ // What is handed back and deployed is the text of the built files, so the fingerprint is
743
+ // worked out here from that text, by the adapter's own rule, and not taken from the job.
744
+ const files = Object.create(null);
745
+ for (const [path, content] of built)
746
+ files[path] = content.toString("utf8");
747
+ const fingerprint = builtFilesFingerprint(files);
748
+ if (fingerprint !== result.built.fingerprint) {
749
+ this.options.log?.(`[runner] build-and-test job ${jobId} for ${request.appId}: the job's fingerprint is not the one the built files have`);
750
+ return failed("The build finished, and the fingerprint the job gave for its files is not theirs, so the files were not kept.");
751
+ }
752
+ const record = {
753
+ appId: request.appId,
754
+ commit: request.commit,
755
+ inputDigest: request.inputDigest,
756
+ fingerprint,
757
+ filesDigest: builtFilesDigest(built),
758
+ files: built.size,
759
+ bytes,
760
+ status: result.status,
761
+ toolsVersion: result.tools.version,
762
+ builtAt: (this.options.now?.() ?? new Date()).toISOString(),
763
+ };
764
+ try {
765
+ recordGadgetBuild(this.options.localStateDir, record);
766
+ }
767
+ catch (error) {
768
+ // Without the record this machine would later refuse to deploy these files, so say so now.
769
+ return failed(`The build finished, and this machine could not record it: ${error instanceof Error ? error.message.slice(0, 300) : String(error)}`);
770
+ }
771
+ return { ...base, result, built: { fingerprint, files }, record };
772
+ }
773
+ }
774
+ /** Reads a JSON request body up to the job's own limit, which is larger than the runner's usual one. */
775
+ export function readGadgetJobBody(req) {
776
+ return new Promise((resolvePromise, reject) => {
777
+ const chunks = [];
778
+ let bytes = 0;
779
+ let refused = false;
780
+ req.on("data", (chunk) => {
781
+ if (refused)
782
+ return;
783
+ bytes += chunk.length;
784
+ if (bytes > GADGET_JOB_LIMITS.maxRequestBytes) {
785
+ refused = true;
786
+ chunks.length = 0;
787
+ reject(new GadgetJobRequestError("The request is larger than a build-and-test job accepts.", 413, "gadget_job_too_large"));
788
+ return;
789
+ }
790
+ chunks.push(chunk);
791
+ });
792
+ req.on("end", () => {
793
+ if (refused)
794
+ return;
795
+ try {
796
+ resolvePromise(JSON.parse(Buffer.concat(chunks).toString("utf8")));
797
+ }
798
+ catch {
799
+ reject(new GadgetJobRequestError("Invalid JSON", 400, "invalid_gadget_job"));
800
+ }
801
+ });
802
+ req.on("error", reject);
803
+ });
804
+ }