@specific.dev/spectest 0.24.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/aws-sigv4.d.ts +42 -0
  2. package/dist/aws-sigv4.js +166 -0
  3. package/dist/browser.d.ts +314 -0
  4. package/dist/browser.js +1320 -0
  5. package/dist/components/email.d.ts +135 -0
  6. package/dist/components/email.js +271 -0
  7. package/dist/components/expo.d.ts +69 -0
  8. package/dist/components/expo.js +125 -0
  9. package/dist/components/index.d.ts +8 -0
  10. package/dist/components/index.js +18 -0
  11. package/dist/components/k3s.d.ts +143 -0
  12. package/dist/components/k3s.js +1067 -0
  13. package/dist/components/postgres.d.ts +93 -0
  14. package/dist/components/postgres.js +58 -0
  15. package/dist/components/replayFake.d.ts +169 -0
  16. package/dist/components/replayFake.js +738 -0
  17. package/dist/components/s3.d.ts +99 -0
  18. package/dist/components/s3.js +81 -0
  19. package/dist/components/supabase.d.ts +197 -0
  20. package/dist/components/supabase.js +1003 -0
  21. package/dist/daemon.d.ts +1 -0
  22. package/dist/daemon.js +4223 -0
  23. package/dist/ids.d.ts +2 -0
  24. package/{src/ids.ts → dist/ids.js} +46 -50
  25. package/dist/index.d.ts +1183 -0
  26. package/dist/index.js +769 -0
  27. package/dist/ingress.d.ts +114 -0
  28. package/dist/ingress.js +210 -0
  29. package/dist/inspect.d.ts +228 -0
  30. package/dist/inspect.js +429 -0
  31. package/dist/locator.d.ts +260 -0
  32. package/dist/locator.js +293 -0
  33. package/dist/mobile.d.ts +71 -0
  34. package/dist/mobile.js +65 -0
  35. package/dist/record-secrets.d.ts +9 -0
  36. package/{src/record-secrets.ts → dist/record-secrets.js} +13 -15
  37. package/dist/recorder.d.ts +516 -0
  38. package/dist/recorder.js +219 -0
  39. package/dist/redis.d.ts +54 -0
  40. package/dist/redis.js +126 -0
  41. package/dist/replay-bundle.d.ts +38 -0
  42. package/{src/replay-bundle.ts → dist/replay-bundle.js} +29 -47
  43. package/dist/resolver.d.ts +1 -0
  44. package/dist/resolver.js +309 -0
  45. package/dist/s3.d.ts +89 -0
  46. package/dist/s3.js +198 -0
  47. package/dist/sql.d.ts +74 -0
  48. package/dist/sql.js +151 -0
  49. package/dist/terminal.d.ts +161 -0
  50. package/dist/terminal.js +538 -0
  51. package/package.json +24 -9
  52. package/src/browser.ts +0 -1807
  53. package/src/components/email.ts +0 -398
  54. package/src/components/expo.ts +0 -167
  55. package/src/components/index.ts +0 -63
  56. package/src/components/k3s.ts +0 -1312
  57. package/src/components/postgres.ts +0 -105
  58. package/src/components/replayFake.ts +0 -848
  59. package/src/components/s3.ts +0 -132
  60. package/src/components/supabase.ts +0 -1299
  61. package/src/daemon.ts +0 -4969
  62. package/src/index.ts +0 -2350
  63. package/src/ingress.ts +0 -288
  64. package/src/inspect.ts +0 -673
  65. package/src/locator.ts +0 -594
  66. package/src/mobile.ts +0 -133
  67. package/src/recorder.ts +0 -817
  68. package/src/redis.ts +0 -202
  69. package/src/resolver.ts +0 -351
  70. package/src/s3.ts +0 -333
  71. package/src/sql.ts +0 -243
  72. package/src/terminal.ts +0 -740
  73. package/src/vendor/rrweb-plugin-console-record.umd.js +0 -521
  74. package/src/vendor/rrweb-record.min.js +0 -5061
package/dist/daemon.js ADDED
@@ -0,0 +1,4223 @@
1
+ // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
2
+ // a container). Owns:
3
+ //
4
+ // * Loading the user's `spectest/index.ts` and exposing the parsed
5
+ // `{ environment, tests? }` to the control plane.
6
+ // * Orchestrating Docker: image prep (pull/build), network + volumes,
7
+ // container start, ready probes — all by shelling out to the local
8
+ // `docker` CLI.
9
+ // * Running individual test cases on demand.
10
+ //
11
+ // The control plane handles VM lifecycle (create, snapshot, fork,
12
+ // terminate) and the tarball uploads to `/workspace` and `/opt/spectest/app`.
13
+ // Once those are in place the daemon does the rest.
14
+ //
15
+ // One sandbox = one daemon. Concurrency between tests is achieved by
16
+ // forking the sandbox; inside a single daemon we never run two tests at
17
+ // once — that keeps stdout capture and timeouts simple.
18
+ import http from "node:http";
19
+ import { execFile, spawn } from "node:child_process";
20
+ import { randomUUID } from "node:crypto";
21
+ import { existsSync, promises as fs, readFileSync } from "node:fs";
22
+ import net from "node:net";
23
+ import path from "node:path";
24
+ import { pathToFileURL } from "node:url";
25
+ import { assert, expect, expectRaw, lowerIngress, dnsName as makeDnsDecl, isWildcard, proxy as makeProxyDecl, } from "./index.js";
26
+ import { acquirePersistentBrowser } from "./browser.js";
27
+ import { isMobileApp, openPersistentMobile } from "./mobile.js";
28
+ import { openTerminal } from "./terminal.js";
29
+ import { recordEnv, recordExec, recordFake, recordHttp, recordTerminal, recordWait, reserveEvent, recorderEventCount, recorderMarkChildren, recorderTruncate, startRecording, stopRecording, truncateUtf8, } from "./recorder.js";
30
+ import { wrap, wrapResponse } from "./inspect.js";
31
+ import { clearRecordSecrets, setRecordSecrets } from "./record-secrets.js";
32
+ import { encodeReplayBundle, replayChunk } from "./replay-bundle.js";
33
+ function namedServices(cfg) {
34
+ return Object.entries(cfg.services).map(([name, def]) => ({ name, ...def }));
35
+ }
36
+ const DEFAULT_PORT = 9876;
37
+ const DEFAULT_TEST_TIMEOUT_MS = 60_000;
38
+ const NETWORK_NAME = process.env.SPECTEST_NETWORK ?? "spectest-net";
39
+ const WORKSPACE = process.env.SPECTEST_WORKSPACE ?? "/workspace";
40
+ // Stable hostname every service container resolves to the host (the
41
+ // `spectest-br0` gateway) — so apps that build or pull images at runtime
42
+ // can point a builder at `spectest-host:5000` (the zot Docker Hub mirror)
43
+ // or `spectest-host:1234` (the shared buildkitd) without hard-coding the
44
+ // gateway IP. Injected into each container's /etc/hosts in runContainer.
45
+ const SPECTEST_HOST_NAME = "spectest-host";
46
+ // The host image-cache gateway, discovered once from the same
47
+ // `registry-mirrors` entry the in-VM dockerd already uses (baked into the
48
+ // local provider's golden /etc/docker/daemon.json). `null` when there's
49
+ // no host cache, so nothing is injected.
50
+ let _hostCacheGateway;
51
+ function hostCacheGateway() {
52
+ if (_hostCacheGateway !== undefined)
53
+ return _hostCacheGateway;
54
+ try {
55
+ const cfg = JSON.parse(readFileSync("/etc/docker/daemon.json", "utf8"));
56
+ const first = cfg["registry-mirrors"]?.[0];
57
+ _hostCacheGateway = first ? new URL(first).hostname || null : null;
58
+ }
59
+ catch {
60
+ _hostCacheGateway = null;
61
+ }
62
+ return _hostCacheGateway;
63
+ }
64
+ const APP_DIR = process.env.SPECTEST_APP_DIR ?? "/opt/spectest/app";
65
+ // Root CA baked into the base snapshot at base-snapshot build time
66
+ // (see base.rs::BASE_SETUP_SH). Bind-mounted into every service
67
+ // container so apps can verify HTTPS to the daemon's fakes, and
68
+ // referenced when we layer it into each image's system trust store.
69
+ const CA_PATH = process.env.SPECTEST_CA_PATH ?? "/etc/spectest/ca.crt";
70
+ const CA_KEY_PATH = process.env.SPECTEST_CA_KEY_PATH ?? "/etc/spectest/ca.key";
71
+ let loaded = null;
72
+ function casesMetadata(suite) {
73
+ if (!suite)
74
+ return [];
75
+ return suite.tests.map((t) => ({
76
+ id: t.id,
77
+ name: t.name,
78
+ dependsOn: t.dependsOn?.id,
79
+ timeoutMs: t.timeoutMs,
80
+ }));
81
+ }
82
+ // Display-only summary of the project's fakes for the control plane (folded
83
+ // into the env config's `fakes`, surfaced on the run page). Fakes aren't part
84
+ // of `project.environment`, so they ride alongside it as a separate field of
85
+ // the /load response — just the routing surface, no handler/state/helpers.
86
+ function fakesSummary(project) {
87
+ if (!project?.fakes)
88
+ return [];
89
+ return Object.entries(project.fakes).map(([name, def]) => ({
90
+ name,
91
+ hostnames: def.hostnames.map((h) => h.toLowerCase()),
92
+ port: def.port ?? DEFAULT_FAKE_PORT,
93
+ }));
94
+ }
95
+ function resolveEntry() {
96
+ const explicit = process.env.SPECTEST_PROJECT_ENTRY;
97
+ if (explicit && existsSync(explicit))
98
+ return explicit;
99
+ const dir = process.env.SPECTEST_PROJECT_DIR ?? path.join(APP_DIR, "spectest");
100
+ for (const name of ["index.ts", "index.mts", "index.mjs", "index.js"]) {
101
+ const p = path.join(dir, name);
102
+ if (existsSync(p))
103
+ return p;
104
+ }
105
+ throw new Error(`could not find project entry in ${dir} (looked for index.ts/.mts/.mjs/.js)`);
106
+ }
107
+ // Import the env entry (`spectest/index.ts`) and load everything that
108
+ // defines the *environment* — services, fakes, project setup — but not the
109
+ // test bodies (those live in `spectest/tests/**`; see `loadTests`). The
110
+ // import URL is deliberately stable (no cache-busting query): test files
111
+ // import this same module (`import { env } from "../index"`) and must resolve
112
+ // to the SAME `env` instance, so their `env.test(...)` calls land in the
113
+ // registry the default-exported Project reads back. Each daemon process
114
+ // imports the entry at most once — an env change always takes the cold path
115
+ // with a fresh daemon — so there's nothing to bust.
116
+ async function loadEnv() {
117
+ const entry = resolveEntry();
118
+ const url = pathToFileURL(entry).href;
119
+ const mod = await import(url);
120
+ const candidate = (mod && typeof mod === "object" && "default" in mod ? mod.default : mod);
121
+ if (!candidate || !candidate.environment) {
122
+ throw new Error(`project entry ${entry} must default-export a Project (from env.project(...) where env = defineEnvironment(...))`);
123
+ }
124
+ loaded = { project: candidate, byId: new Map() };
125
+ rebuildCatalogue();
126
+ // Any cached convenience clients belong to the previous project; drop
127
+ // them so the next test rebuilds against the freshly loaded definitions.
128
+ HELPERS_CACHE.clear();
129
+ // Register fakes + service-tls proxies here; startIngress() during
130
+ // /bootstrap actually binds the listeners — this just parses + validates
131
+ // and tears down any prior runtime so a reload picks up edits.
132
+ buildIngress(candidate);
133
+ return candidate;
134
+ }
135
+ // Import the test files under `spectest/tests/**` into the already-loaded
136
+ // env, then refresh the catalogue. The split layout keeps test bodies out of
137
+ // the warm-template cache key, so a test-only edit restores the cached env
138
+ // and lands here to pick up the new tests. A no-op for the legacy single-file
139
+ // layout (no `tests/` dir; the suite is already on the default export).
140
+ //
141
+ // Crucially this is only ever called AFTER the warm-template snapshot is
142
+ // captured (cold path) or against a freshly restored VM (warm path), so each
143
+ // test file is imported for the first time in that daemon process — no
144
+ // cache-busting needed and no stale ESM module to fight.
145
+ async function loadTests() {
146
+ requireLoaded();
147
+ const dir = path.join(path.dirname(resolveEntry()), "tests");
148
+ if (existsSync(dir)) {
149
+ for (const file of await collectTestFiles(dir)) {
150
+ await import(pathToFileURL(file).href);
151
+ }
152
+ }
153
+ rebuildCatalogue();
154
+ // The test set may have changed; drop cached helpers so the next test
155
+ // rebuilds cleanly.
156
+ HELPERS_CACHE.clear();
157
+ }
158
+ // (Re)build the id→TestCase index from whatever the loaded project currently
159
+ // exposes as its suite: the lazy registry getter for split layouts, or the
160
+ // frozen explicit suite for inline ones.
161
+ function rebuildCatalogue() {
162
+ const l = requireLoaded();
163
+ const byId = new Map();
164
+ if (l.project.tests) {
165
+ for (const t of l.project.tests.tests)
166
+ byId.set(t.id, t);
167
+ }
168
+ l.byId = byId;
169
+ }
170
+ // Recursively collect importable test modules under `spectest/tests/`, sorted
171
+ // for deterministic import order. Skips declaration files and dependency dirs.
172
+ async function collectTestFiles(dir) {
173
+ const out = [];
174
+ const walk = async (d) => {
175
+ const entries = await fs.readdir(d, { withFileTypes: true });
176
+ for (const e of entries) {
177
+ const full = path.join(d, e.name);
178
+ if (e.isDirectory()) {
179
+ if (e.name === "node_modules" || e.name === ".spectest")
180
+ continue;
181
+ await walk(full);
182
+ }
183
+ else if (e.isFile() &&
184
+ /\.(ts|mts|cts|js|mjs|cjs)$/.test(e.name) &&
185
+ !e.name.endsWith(".d.ts")) {
186
+ out.push(full);
187
+ }
188
+ }
189
+ };
190
+ await walk(dir);
191
+ out.sort();
192
+ return out;
193
+ }
194
+ function requireLoaded() {
195
+ if (!loaded) {
196
+ throw new Error("no project loaded; call POST /load first");
197
+ }
198
+ return loaded;
199
+ }
200
+ function shx(file, args, timeoutMs, env) {
201
+ return new Promise((resolve) => {
202
+ let done = false;
203
+ const child = execFile(file, args, { maxBuffer: 64 * 1024 * 1024, env: env ? { ...process.env, ...env } : process.env }, (err, stdout, stderr) => {
204
+ if (done)
205
+ return;
206
+ done = true;
207
+ const code = err && typeof err.code === "number"
208
+ ? Number(err.code)
209
+ : err
210
+ ? 1
211
+ : 0;
212
+ resolve({ stdout: String(stdout), stderr: String(stderr), code });
213
+ });
214
+ if (timeoutMs && timeoutMs > 0) {
215
+ setTimeout(() => {
216
+ if (!done) {
217
+ done = true;
218
+ try {
219
+ child.kill("SIGKILL");
220
+ }
221
+ catch {
222
+ /* already exited */
223
+ }
224
+ resolve({ stdout: "", stderr: `timeout after ${timeoutMs}ms`, code: 124 });
225
+ }
226
+ }, timeoutMs);
227
+ }
228
+ });
229
+ }
230
+ function docker(args, timeoutMs, env) {
231
+ return shx("docker", args, timeoutMs, env);
232
+ }
233
+ /**
234
+ * Like `shx` but invokes `onLine` for each line of combined stdout/stderr
235
+ * as it streams in, so callers can surface live progress (docker build
236
+ * steps, image pull layers) into bootstrap progress. Still resolves with
237
+ * the full captured output + exit code, so existing error handling and
238
+ * post-hoc parsing (`summarizeBuildKit`) are unchanged.
239
+ */
240
+ function shxStream(file, args, timeoutMs, env, onLine) {
241
+ return new Promise((resolve) => {
242
+ const child = spawn(file, args, {
243
+ env: env ? { ...process.env, ...env } : process.env,
244
+ });
245
+ let stdout = "";
246
+ let stderr = "";
247
+ let buf = "";
248
+ let done = false;
249
+ const feed = (chunk) => {
250
+ buf += chunk;
251
+ let nl;
252
+ while ((nl = buf.indexOf("\n")) >= 0) {
253
+ const line = buf.slice(0, nl).replace(/\r$/, "");
254
+ buf = buf.slice(nl + 1);
255
+ try {
256
+ onLine(line);
257
+ }
258
+ catch {
259
+ /* a progress callback must never break the build */
260
+ }
261
+ }
262
+ };
263
+ child.stdout?.on("data", (d) => {
264
+ const s = String(d);
265
+ stdout += s;
266
+ feed(s);
267
+ });
268
+ child.stderr?.on("data", (d) => {
269
+ const s = String(d);
270
+ stderr += s;
271
+ feed(s);
272
+ });
273
+ const finish = (code, extraStderr) => {
274
+ if (done)
275
+ return;
276
+ done = true;
277
+ resolve({ stdout, stderr: extraStderr ? stderr + extraStderr : stderr, code });
278
+ };
279
+ child.on("error", () => finish(1));
280
+ child.on("close", (code) => finish(code == null ? 1 : code));
281
+ if (timeoutMs && timeoutMs > 0) {
282
+ setTimeout(() => {
283
+ if (done)
284
+ return;
285
+ try {
286
+ child.kill("SIGKILL");
287
+ }
288
+ catch {
289
+ /* already exited */
290
+ }
291
+ finish(124, `\ntimeout after ${timeoutMs}ms`);
292
+ }, timeoutMs);
293
+ }
294
+ });
295
+ }
296
+ let BOOTSTRAP_PROGRESS = null;
297
+ function progressInit(services) {
298
+ BOOTSTRAP_PROGRESS = {
299
+ phase: "Preparing images",
300
+ services: services.map((s) => ({
301
+ name: s.name,
302
+ kind: s.image.type === "registry" ? "pull" : "build",
303
+ status: "pending",
304
+ })),
305
+ startedAt: Date.now(),
306
+ updatedAt: Date.now(),
307
+ done: false,
308
+ };
309
+ }
310
+ function progressPhase(phase) {
311
+ if (!BOOTSTRAP_PROGRESS)
312
+ return;
313
+ BOOTSTRAP_PROGRESS.phase = phase;
314
+ BOOTSTRAP_PROGRESS.updatedAt = Date.now();
315
+ }
316
+ function progressService(name, patch) {
317
+ if (!BOOTSTRAP_PROGRESS)
318
+ return;
319
+ const svc = BOOTSTRAP_PROGRESS.services.find((s) => s.name === name);
320
+ if (!svc)
321
+ return;
322
+ Object.assign(svc, patch);
323
+ BOOTSTRAP_PROGRESS.updatedAt = Date.now();
324
+ }
325
+ function progressDone() {
326
+ if (!BOOTSTRAP_PROGRESS)
327
+ return;
328
+ BOOTSTRAP_PROGRESS.phase = "Ready";
329
+ BOOTSTRAP_PROGRESS.done = true;
330
+ BOOTSTRAP_PROGRESS.updatedAt = Date.now();
331
+ }
332
+ // BuildKit (docker buildx) gives per-step timing via `--progress=plain`,
333
+ // parallel stages, and `RUN --mount=type=cache`. Detected once: where the
334
+ // buildx plugin isn't installed (e.g. a Freestyle base without it) we fall
335
+ // back to the legacy builder, which takes no `--progress` flag.
336
+ let _buildxAvailable;
337
+ async function hasBuildx() {
338
+ if (_buildxAvailable === undefined) {
339
+ const r = await docker(["buildx", "version"], 15_000);
340
+ _buildxAvailable = r.code === 0;
341
+ }
342
+ return _buildxAvailable;
343
+ }
344
+ // A single buildkitd runs on the host (see scripts/install-buildkitd.sh),
345
+ // reachable from every VM at the bridge gateway. Building against it as a
346
+ // `remote` buildx builder gives a persistent, shared layer/mount cache that
347
+ // survives forks and warm-template misses — a fresh VM no longer rebuilds
348
+ // from scratch. The build runs on the host (runc-isolated); `--load` pulls
349
+ // the finished image back into the in-VM dockerd. Detected once; if the
350
+ // builder can't be created or buildkitd is unreachable we fall back to the
351
+ // in-VM builder, so a missing/dead buildkitd just means slower builds.
352
+ const REMOTE_BUILDER_ADDR = process.env.SPECTEST_BUILDKIT_ADDR ?? "tcp://10.42.0.1:1234";
353
+ const REMOTE_BUILDER_NAME = "spectest-remote";
354
+ let _remoteBuilder;
355
+ async function ensureRemoteBuilder() {
356
+ if (_remoteBuilder !== undefined)
357
+ return _remoteBuilder;
358
+ if (!(await hasBuildx())) {
359
+ _remoteBuilder = false;
360
+ return false;
361
+ }
362
+ // Idempotent: a repeat create with the same name errors ("existing
363
+ // instance"), which we treat as already-present.
364
+ const create = await docker(["buildx", "create", "--name", REMOTE_BUILDER_NAME, "--driver", "remote", REMOTE_BUILDER_ADDR], 30_000);
365
+ if (create.code !== 0 && !/existing instance|already exists/i.test(create.stderr)) {
366
+ _remoteBuilder = false;
367
+ return false;
368
+ }
369
+ // `inspect --bootstrap` actually dials buildkitd, so it's our reachability
370
+ // probe. If buildkitd is down this fails and we fall back.
371
+ const boot = await docker(["buildx", "inspect", "--bootstrap", REMOTE_BUILDER_NAME], 60_000);
372
+ _remoteBuilder = boot.code === 0;
373
+ if (!_remoteBuilder) {
374
+ // eslint-disable-next-line no-console
375
+ console.warn(`[build] remote buildkitd at ${REMOTE_BUILDER_ADDR} unreachable; using in-VM builder:\n${boot.stderr.trim()}`);
376
+ }
377
+ return _remoteBuilder;
378
+ }
379
+ // Parse `docker build --progress=plain` (BuildKit) output into per-step
380
+ // timings, sorted slowest-first. Steps are correlated by their `#N` id:
381
+ // the declaration line carries the command, the `DONE`/`CACHED` line the
382
+ // duration. Best-effort — unparseable output yields an empty list.
383
+ function summarizeBuildKit(out) {
384
+ const names = new Map();
385
+ const secs = new Map();
386
+ const cached = new Set();
387
+ for (const line of out.split("\n")) {
388
+ let m = line.match(/^#(\d+)\s+\[[^\]]*\]\s+(.+)$/);
389
+ if (m) {
390
+ const id = `#${m[1]}`;
391
+ if (!names.has(id))
392
+ names.set(id, m[2].trim().slice(0, 80));
393
+ continue;
394
+ }
395
+ m = line.match(/^#(\d+)\s+DONE\s+([\d.]+)s/);
396
+ if (m) {
397
+ secs.set(`#${m[1]}`, parseFloat(m[2]));
398
+ continue;
399
+ }
400
+ m = line.match(/^#(\d+)\s+CACHED/);
401
+ if (m) {
402
+ const id = `#${m[1]}`;
403
+ cached.add(id);
404
+ if (!secs.has(id))
405
+ secs.set(id, 0);
406
+ }
407
+ }
408
+ const steps = [];
409
+ for (const [id, name] of names) {
410
+ steps.push({ name, secs: secs.get(id) ?? 0, cached: cached.has(id) });
411
+ }
412
+ return steps.sort((a, b) => b.secs - a.secs);
413
+ }
414
+ // ────────────────────────────────────────────────────────────────────────
415
+ // Bootstrap stages
416
+ // ────────────────────────────────────────────────────────────────────────
417
+ async function ensureNetwork() {
418
+ const inspect = await docker(["network", "inspect", NETWORK_NAME], 30_000);
419
+ if (inspect.code === 0)
420
+ return;
421
+ const create = await docker(["network", "create", NETWORK_NAME], 60_000);
422
+ if (create.code !== 0) {
423
+ throw new Error(`docker network create ${NETWORK_NAME} failed: ${create.stderr.trim() || create.stdout.trim()}`);
424
+ }
425
+ }
426
+ function sanitizeSegment(p) {
427
+ return p
428
+ .replace(/^\/+/, "")
429
+ .replace(/[^A-Za-z0-9_-]/g, "-")
430
+ .replace(/^-+|-+$/g, "");
431
+ }
432
+ function resolveHostPath(service, vol) {
433
+ if (vol.name) {
434
+ // Named shared volume: one backing dir per name, shared by every
435
+ // service that mounts the same name (storage-api ↔ imgproxy). Rooted
436
+ // in the per-env state tree (or the cache tree when cache-flagged),
437
+ // so teardown/fork semantics match ordinary volumes.
438
+ const root = vol.cache
439
+ ? ["/var/cache/spectest/volumes", "_shared"]
440
+ : [WORKSPACE, ".spectest", "volumes", "_shared"];
441
+ return path.join(...root, sanitizeSegment(vol.name));
442
+ }
443
+ if (vol.source && vol.source.startsWith("/"))
444
+ return vol.source;
445
+ // Cache volumes root OUTSIDE /workspace so the delta-restore teardown
446
+ // (rm -rf /workspace) keeps them — they hold only content-addressed
447
+ // accelerator data (see VolumeMount.cache), never env state.
448
+ const root = vol.cache
449
+ ? ["/var/cache/spectest/volumes", service]
450
+ : [WORKSPACE, ".spectest", "volumes", service];
451
+ if (vol.source) {
452
+ return path.join(...root, vol.source.replace(/^\/+/, ""));
453
+ }
454
+ return path.join(...root, sanitizeSegment(vol.target));
455
+ }
456
+ /// Where the daemon records every ABSOLUTE-source, non-cache volume dir it
457
+ /// has created, one path per line. The delta-restore teardown wipes the
458
+ /// listed dirs: they live outside /workspace (which the teardown removes
459
+ /// wholesale) and outside /var/cache/spectest (deliberately kept), so
460
+ /// without this manifest a `source: "/data/pg"` volume would carry the
461
+ /// previous generation's data into a "fresh" environment. tmpfs-backed
462
+ /// (/run) — survives snapshots like all guest memory, dies with the VM.
463
+ const VOLUME_DIRS_MANIFEST = "/run/spectest-volume-dirs";
464
+ const ABS_VOLUME_DIRS = new Set();
465
+ async function recordAbsoluteVolumeDir(host) {
466
+ if (ABS_VOLUME_DIRS.has(host))
467
+ return;
468
+ ABS_VOLUME_DIRS.add(host);
469
+ await fs.writeFile(VOLUME_DIRS_MANIFEST, [...ABS_VOLUME_DIRS].join("\n") + "\n");
470
+ }
471
+ async function ensureVolumes(svc) {
472
+ const flags = [];
473
+ if (!svc.volumes || svc.volumes.length === 0)
474
+ return flags;
475
+ for (const vol of svc.volumes) {
476
+ // Boot services are validated in defineEnvironment; re-check here so
477
+ // runtime `startService` specs get the same contract.
478
+ if (vol.name && vol.source) {
479
+ throw new Error(`service "${svc.name}" volume for ${JSON.stringify(vol.target)} sets both \`name\` and \`source\``);
480
+ }
481
+ const host = resolveHostPath(svc.name, vol);
482
+ await fs.mkdir(host, { recursive: true });
483
+ if (vol.source?.startsWith("/") && !host.startsWith("/var/cache/spectest/")) {
484
+ await recordAbsoluteVolumeDir(host);
485
+ }
486
+ flags.push(`--volume=${host}:${vol.target}${vol.readOnly ? ":ro" : ""}`);
487
+ }
488
+ return flags;
489
+ }
490
+ // Materialize `svc.files` onto the VM host and return `--volume` flags
491
+ // bind-mounting each into the container (read-only). Single-file bind
492
+ // mounts mean the seeded config lands in place *before the container's
493
+ // entrypoint runs* — the one injection point earlier than any setup
494
+ // hook. Staging path mirrors ensureVolumes: a per-service dir derived
495
+ // from the in-container path, so two files never collide and the
496
+ // content is captured by snapshots like everything else under WORKSPACE.
497
+ async function ensureFiles(svc) {
498
+ const flags = [];
499
+ if (!svc.files || svc.files.length === 0)
500
+ return flags;
501
+ const dir = path.join(WORKSPACE, ".spectest", "files", svc.name);
502
+ await fs.mkdir(dir, { recursive: true });
503
+ for (const f of svc.files) {
504
+ if (!f.path.startsWith("/")) {
505
+ throw new Error(`service "${svc.name}": file path ${JSON.stringify(f.path)} must be absolute`);
506
+ }
507
+ // `{{SPECTEST_SERVICE}}` expands to this service's name (its
508
+ // services-map key) so a component can author self-referential
509
+ // config without knowing the key the user will choose — e.g. k3s's
510
+ // registries.yaml keying on `<key>.internal:5000`.
511
+ const content = f.content.replaceAll("{{SPECTEST_SERVICE}}", svc.name);
512
+ const host = path.join(dir, sanitizeSegment(f.path));
513
+ await fs.writeFile(host, content);
514
+ if (f.mode)
515
+ await fs.chmod(host, parseInt(f.mode, 8));
516
+ flags.push(`--volume=${host}:${f.path}:ro`);
517
+ }
518
+ return flags;
519
+ }
520
+ function imageTag(name) {
521
+ return `spectest/${name}:latest`;
522
+ }
523
+ const DEFAULT_DOCKERIGNORE = [
524
+ ".git",
525
+ ".spectest",
526
+ "spectest",
527
+ "node_modules",
528
+ "target",
529
+ "__pycache__",
530
+ ".venv",
531
+ ".env",
532
+ ".env.local",
533
+ ".env.*",
534
+ "dist",
535
+ "build",
536
+ ".next",
537
+ ".turbo",
538
+ ".DS_Store",
539
+ ];
540
+ function unionDockerignore(services) {
541
+ const seen = new Set(DEFAULT_DOCKERIGNORE);
542
+ const extras = [];
543
+ for (const s of services) {
544
+ if (s.image.type === "dockerfile" && s.image.exclude) {
545
+ for (const e of s.image.exclude) {
546
+ if (!seen.has(e)) {
547
+ seen.add(e);
548
+ extras.push(e);
549
+ }
550
+ }
551
+ }
552
+ }
553
+ return [...DEFAULT_DOCKERIGNORE, ...extras].join("\n") + "\n";
554
+ }
555
+ /// In-flight/finished dockerfile builds of this bootstrap, keyed by
556
+ /// sha256(dockerfile content + exclude list). Services that share an
557
+ /// identical image definition (e.g. an API server and a worker running the
558
+ /// same codebase with different entrypoints) build ONCE; the others wait
559
+ /// and `docker tag` the result. Cleared at every bootstrap() — the build
560
+ /// CONTEXT (/workspace) is an input too, so dedup is only valid within one
561
+ /// workspace generation (runtime services started mid-test share it).
562
+ const BUILD_DEDUP = new Map();
563
+ function buildContentKey(image) {
564
+ return new Bun.CryptoHasher("sha256")
565
+ .update(image.content)
566
+ .update("\0")
567
+ .update(JSON.stringify(image.exclude ?? []))
568
+ .digest("hex");
569
+ }
570
+ async function prepareServiceImage(svc, opts) {
571
+ const tag = imageTag(svc.name);
572
+ if (svc.image.type === "registry") {
573
+ const ref = svc.image.reference;
574
+ // Always pull — even when the (delta-restored) store already has the
575
+ // ref. With the layers present this costs ~a manifest round-trip per
576
+ // image ("Already exists" all the way down, through the zot mirror),
577
+ // off the bootstrap critical path; skipping it would freeze floating
578
+ // tags (`foo:latest`) at whatever the previous generation pulled, for
579
+ // as long as the delta chain lives — a silent semantic divergence
580
+ // from the cold build a delta restore must be equivalent to.
581
+ progressService(svc.name, { status: "pulling", detail: `pulling ${ref}` });
582
+ let layers = 0;
583
+ const pull = await shxStream("docker", ["pull", ref], 900_000, undefined, (line) => {
584
+ // `docker pull` (no TTY) prints one line per layer: "<id>: Pull
585
+ // complete" / "Already exists". Count them for a live layer tally.
586
+ if (/(?:Pull complete|Already exists)\s*$/.test(line)) {
587
+ layers++;
588
+ progressService(svc.name, { status: "pulling", detail: `${layers} layers` });
589
+ }
590
+ });
591
+ if (pull.code !== 0) {
592
+ progressService(svc.name, { status: "failed" });
593
+ throw new Error(`docker pull ${ref} failed: ${pull.stderr.trim() || pull.stdout.trim()}`);
594
+ }
595
+ const tagr = await docker(["tag", ref, tag], 30_000);
596
+ if (tagr.code !== 0) {
597
+ throw new Error(`docker tag ${ref} ${tag} failed: ${tagr.stderr.trim()}`);
598
+ }
599
+ await ensureCaTrustedImage(svc.name, tag);
600
+ return { tag };
601
+ }
602
+ // Dockerfile build. Within one bootstrap, identical definitions (shared
603
+ // codebase images) dedup to a single build. Only bootstrap opts in: the
604
+ // dedup key is dockerfile content + exclude, but the build CONTEXT
605
+ // (/workspace) is an input too — a runtime service started mid-test
606
+ // after setup/test code mutated /workspace must rebuild, not share a
607
+ // pre-mutation image.
608
+ const image = svc.image;
609
+ if (opts?.dedup) {
610
+ const key = buildContentKey(image);
611
+ const inflight = BUILD_DEDUP.get(key);
612
+ if (inflight) {
613
+ progressService(svc.name, { status: "building", detail: `sharing ${inflight.name}'s build` });
614
+ let first;
615
+ try {
616
+ first = await inflight.promise;
617
+ }
618
+ catch (err) {
619
+ progressService(svc.name, { status: "failed" });
620
+ throw err;
621
+ }
622
+ // first.tag is already CA-layered; tagging it covers this service too.
623
+ const tagr = await docker(["tag", first.tag, tag], 30_000);
624
+ if (tagr.code !== 0) {
625
+ throw new Error(`docker tag ${first.tag} ${tag} failed: ${tagr.stderr.trim()}`);
626
+ }
627
+ progressService(svc.name, { status: "prepared" });
628
+ return { tag, buildSteps: first.buildSteps };
629
+ }
630
+ const promise = buildServiceImage(svc.name, image, tag);
631
+ BUILD_DEDUP.set(key, { name: svc.name, promise });
632
+ try {
633
+ return await promise;
634
+ }
635
+ catch (err) {
636
+ // Let a sharer arriving later rebuild rather than inherit this
637
+ // build's failure forever.
638
+ BUILD_DEDUP.delete(key);
639
+ throw err;
640
+ }
641
+ }
642
+ return buildServiceImage(svc.name, image, tag);
643
+ }
644
+ async function buildServiceImage(name, image, tag) {
645
+ let buildSteps;
646
+ {
647
+ const dfDir = path.join(WORKSPACE, ".spectest", "services", name);
648
+ await fs.mkdir(dfDir, { recursive: true });
649
+ const dfPath = path.join(dfDir, "Dockerfile");
650
+ await fs.writeFile(dfPath, image.content);
651
+ // Per-service ignore: BuildKit resolves `<Dockerfile>.dockerignore`
652
+ // (next to the Dockerfile) in preference to the context root's
653
+ // `.dockerignore`, so this build sees the defaults plus ITS OWN
654
+ // `exclude` only — one service excluding `handhelds/**` no longer
655
+ // empties a sibling's build context. Verified on both the remote-buildx
656
+ // and DOCKER_BUILDKIT paths (client-side context filtering). The root
657
+ // union `.dockerignore` written at bootstrap stays as the fallback for
658
+ // the legacy non-BuildKit builder, which predates per-Dockerfile
659
+ // ignores.
660
+ await fs.writeFile(`${dfPath}.dockerignore`, [...DEFAULT_DOCKERIGNORE, ...(image.exclude ?? [])].join("\n") + "\n");
661
+ const useRemote = await ensureRemoteBuilder();
662
+ // Both the remote builder and a local buildx are BuildKit, so both emit
663
+ // per-step timing on stderr under `--progress=plain` (parsed below). Only
664
+ // the legacy in-VM builder takes no progress flag.
665
+ const useBuildKit = useRemote || (await hasBuildx());
666
+ const buildEnv = {};
667
+ let buildArgs;
668
+ if (useRemote) {
669
+ // Build on the host-side shared buildkitd (persistent cross-VM cache);
670
+ // `--load` brings the finished image back into the in-VM dockerd so
671
+ // runContainer can `docker run` it. The build context (WORKSPACE, minus
672
+ // .dockerignore) streams to buildkitd over the bridge.
673
+ buildArgs = [
674
+ "buildx", "build",
675
+ "--builder", REMOTE_BUILDER_NAME,
676
+ "--load",
677
+ "--progress=plain",
678
+ "-t", tag, "-f", dfPath, WORKSPACE,
679
+ ];
680
+ }
681
+ else if (useBuildKit) {
682
+ buildArgs = ["build", "-t", tag, "-f", dfPath, "--progress=plain", WORKSPACE];
683
+ buildEnv.DOCKER_BUILDKIT = "1";
684
+ }
685
+ else {
686
+ buildArgs = ["build", "-t", tag, "-f", dfPath, WORKSPACE];
687
+ }
688
+ progressService(name, { status: "building", detail: "starting build" });
689
+ const build = await shxStream("docker", buildArgs, 1_800_000, buildEnv, (line) => {
690
+ // BuildKit `--progress=plain` declares each step as
691
+ // `#N [<stage> M/N] <cmd>`; the legacy builder as `Step M/N : <cmd>`.
692
+ // Track the most-recent step as live detail.
693
+ let m = line.match(/^#\d+\s+\[([^\]]*)\]\s+(.+)$/);
694
+ if (m) {
695
+ const step = m[1].match(/\d+\/\d+/)?.[0];
696
+ const cmd = m[2].trim().slice(0, 60);
697
+ progressService(name, {
698
+ status: "building",
699
+ detail: step ? `step ${step} ${cmd}` : cmd,
700
+ });
701
+ return;
702
+ }
703
+ m = line.match(/^Step (\d+\/\d+)\s*:\s*(.+)$/);
704
+ if (m) {
705
+ progressService(name, {
706
+ status: "building",
707
+ detail: `step ${m[1]} ${m[2].trim().slice(0, 60)}`,
708
+ });
709
+ }
710
+ });
711
+ if (build.code !== 0) {
712
+ progressService(name, { status: "failed" });
713
+ throw new Error(`docker build for ${name} failed:\n${build.stderr.trim()}\n${build.stdout.trim()}`);
714
+ }
715
+ if (useBuildKit) {
716
+ // Keep only the slowest dozen steps ≥1s — enough to profile, small
717
+ // enough to ride back in the /bootstrap response and the journal.
718
+ buildSteps = summarizeBuildKit(build.stderr)
719
+ .filter((s) => s.secs >= 1)
720
+ .slice(0, 12);
721
+ }
722
+ }
723
+ // Layer the spectest CA into the image's system trust store so apps
724
+ // that read the system bundle (Go, Java, CLIs that don't honour the
725
+ // SSL_CERT_FILE env vars) accept HTTPS to fakes. Best-effort: images
726
+ // without `update-ca-certificates` / `update-ca-trust` (distroless,
727
+ // scratch) fall through to the env-var path that `runContainer` sets.
728
+ await ensureCaTrustedImage(name, tag);
729
+ return { tag, buildSteps };
730
+ }
731
+ /**
732
+ * Build a derivative image on top of `tag` that copies the spectest
733
+ * root CA into the system trust store. Tagged back as `tag`, so the
734
+ * rest of the orchestrator (runContainer, image cache) is oblivious.
735
+ * Failures are warned-and-ignored: the env-var injection in
736
+ * runContainer is the universal fallback, so apps that use it (most
737
+ * Node/Python/Ruby/AWS clients) still trust the CA even when the
738
+ * image's trust store can't be updated.
739
+ */
740
+ async function ensureCaTrustedImage(serviceName, tag) {
741
+ if (!existsSync(CA_PATH)) {
742
+ // Daemon running outside a base-snapshot VM (dev/test). Nothing to
743
+ // layer; env vars also harmless (they point at a missing path, but
744
+ // most consumers ignore missing files).
745
+ return;
746
+ }
747
+ const ctxDir = path.join(WORKSPACE, ".spectest", "ca-trust", serviceName);
748
+ await fs.mkdir(ctxDir, { recursive: true });
749
+ await fs.copyFile(CA_PATH, path.join(ctxDir, "spectest-ca.crt"));
750
+ const dockerfile = `FROM ${tag}
751
+ COPY spectest-ca.crt /usr/local/share/ca-certificates/spectest-ca.crt
752
+ RUN if command -v update-ca-certificates >/dev/null 2>&1; then \\
753
+ update-ca-certificates; \\
754
+ elif command -v update-ca-trust >/dev/null 2>&1; then \\
755
+ cp /usr/local/share/ca-certificates/spectest-ca.crt /etc/pki/ca-trust/source/anchors/spectest-ca.crt && update-ca-trust extract; \\
756
+ else \\
757
+ echo "[spectest] no system CA trust tool in image; env-var trust only"; \\
758
+ fi
759
+ `;
760
+ await fs.writeFile(path.join(ctxDir, "Dockerfile"), dockerfile);
761
+ const build = await docker(["build", "-t", tag, ctxDir], 300_000);
762
+ if (build.code !== 0) {
763
+ // eslint-disable-next-line no-console
764
+ console.warn(`[ca-trust] could not layer spectest CA into ${serviceName} (${tag}); env-var fallback only:\n${build.stderr.trim() || build.stdout.trim()}`);
765
+ }
766
+ }
767
+ async function runContainer(svc, tag, volumeFlags,
768
+ // Extra `--network-alias`es beyond the lowered `aliasesByService`. Used by
769
+ // runtime `startService` (whose service isn't in LOWERED) to give the new
770
+ // container docker-native multi-label resolution for its `hostnames`.
771
+ extraAliases = []) {
772
+ // Idempotent: clean up any leftover container with the same name.
773
+ await docker(["rm", "-f", svc.name], 30_000);
774
+ const args = [
775
+ "run",
776
+ "-d",
777
+ "--restart=no",
778
+ `--name=${svc.name}`,
779
+ `--hostname=${svc.name}`,
780
+ `--network=${NETWORK_NAME}`,
781
+ // Every service is reachable at `<name>.internal` as well as its
782
+ // bare `<name>`. The fully-qualified form is what kubeconfigs and
783
+ // other tooling that expect a multi-label hostname should use; it's
784
+ // resolved both inside containers (Docker's embedded DNS) and on
785
+ // the VM host (spectest-resolver scans aliases).
786
+ `--network-alias=${svc.name}.internal`,
787
+ ];
788
+ // Extra peer aliases for this service — lowered from `hostnames` and any
789
+ // dnsName(h, { service }) into LOWERED.aliasesByService, plus any passed
790
+ // explicitly by a runtime startService (not present in LOWERED).
791
+ for (const h of [...(LOWERED.aliasesByService[svc.name] ?? []), ...extraAliases]) {
792
+ args.push(`--network-alias=${h}`);
793
+ }
794
+ // Bound TCP give-up time inside THIS container's network namespace.
795
+ // net.ipv4.tcp_retries2 is per-netns and a fresh netns resets to the kernel
796
+ // default (15 ≈ ~15 min of RTO backoff), so lowering it on the guest's init
797
+ // netns (BASE_SETUP_SH) does NOT reach containers — and the connections that
798
+ // actually wedge run here: buildkit/buildctl pulling base images + exporting
799
+ // cache, and the k3s container's containerd pulling images, all to the host
800
+ // zot over the VM↔host path. On a lost-retransmit (transient loss under
801
+ // concurrent forks) such a flow otherwise stalls a build/pull for minutes.
802
+ // Setting it per container resets a genuinely-stuck flow in ~tens of seconds
803
+ // so the client retries on a fresh connection; live connections keep getting
804
+ // ACKs and are unaffected. Safe because every service runs on the
805
+ // spectest-net bridge (own netns), never --network=host where net.* is denied.
806
+ args.push("--sysctl", "net.ipv4.tcp_retries2=6");
807
+ // Wire every ingress hostname (fakes, TLS-terminated proxies, and any
808
+ // dnsName(h, { ingress: true })) into the container's /etc/hosts so
809
+ // `fetch("http://api.stripe.com")` or `fetch("https://app.test")` from
810
+ // app code reaches the daemon's ingress listener via the bridge gateway.
811
+ // /etc/hosts beats Docker's embedded DNS (127.0.0.11), so we don't need
812
+ // to touch the container's resolver settings.
813
+ if (cachedGatewayIp) {
814
+ for (const h of LOWERED.ingressHosts) {
815
+ args.push(`--add-host=${h}:${cachedGatewayIp}`);
816
+ }
817
+ }
818
+ // Resolve `spectest-host` to the host image-cache gateway so apps can
819
+ // address the zot mirrors / shared buildkitd by name (see
820
+ // SPECTEST_HOST_NAME). Skipped where there's no host cache.
821
+ const hostGw = hostCacheGateway();
822
+ if (hostGw)
823
+ args.push(`--add-host=${SPECTEST_HOST_NAME}:${hostGw}`);
824
+ if (svc.workdir)
825
+ args.push(`--workdir=${svc.workdir}`);
826
+ // Trust the spectest root CA from inside the container. Bind-mount
827
+ // the cert + set the conventional env vars so language runtimes
828
+ // (Node, Python requests/httpx, AWS SDKs) pick it up without
829
+ // touching the image's system trust store. The per-image
830
+ // ensureCaTrustedImage layer also installs it into the system
831
+ // trust store; this env-var path is the belt-and-braces fallback
832
+ // for images where the layer step couldn't run (no
833
+ // update-ca-certificates).
834
+ args.push(`--volume=${CA_PATH}:${CA_PATH}:ro`);
835
+ args.push("-e", `NODE_EXTRA_CA_CERTS=${CA_PATH}`);
836
+ args.push("-e", `SSL_CERT_FILE=${CA_PATH}`);
837
+ args.push("-e", `REQUESTS_CA_BUNDLE=${CA_PATH}`);
838
+ args.push("-e", `AWS_CA_BUNDLE=${CA_PATH}`);
839
+ if (svc.env) {
840
+ for (const [k, v] of Object.entries(svc.env)) {
841
+ args.push("-e", `${k}=${v}`);
842
+ }
843
+ }
844
+ for (const flag of volumeFlags)
845
+ args.push(flag);
846
+ if (svc.privileged)
847
+ args.push("--privileged");
848
+ for (const p of svc.tmpfs ?? [])
849
+ args.push(`--tmpfs=${p}`);
850
+ if (svc.cgroupns)
851
+ args.push(`--cgroupns=${svc.cgroupns}`);
852
+ // `command` runs via sh -c, replacing the image entrypoint; `args` is a
853
+ // plain CMD override (`docker run <image> <args…>`) that keeps the
854
+ // entrypoint — what init-wrapped images (postgres) need for extra flags.
855
+ if (svc.command && svc.args?.length) {
856
+ throw new Error(`service ${svc.name}: \`command\` and \`args\` are mutually exclusive ` +
857
+ `(command replaces the entrypoint with /bin/sh -c; args keeps it)`);
858
+ }
859
+ if (svc.command)
860
+ args.push("--entrypoint=/bin/sh");
861
+ args.push(tag);
862
+ if (svc.command)
863
+ args.push("-c", svc.command);
864
+ else if (svc.args?.length)
865
+ args.push(...svc.args);
866
+ const r = await docker(args, 300_000);
867
+ if (r.code !== 0) {
868
+ throw new Error(`docker run ${svc.name} failed: ${r.stderr.trim() || r.stdout.trim()}`);
869
+ }
870
+ }
871
+ async function probeTcp(host, port) {
872
+ return new Promise((resolve) => {
873
+ const sock = net.createConnection({ host, port });
874
+ let settled = false;
875
+ const finish = (v) => {
876
+ if (settled)
877
+ return;
878
+ settled = true;
879
+ try {
880
+ sock.destroy();
881
+ }
882
+ catch {
883
+ /* ignore */
884
+ }
885
+ resolve(v);
886
+ };
887
+ sock.setTimeout(2000);
888
+ sock.once("connect", () => finish(true));
889
+ sock.once("error", () => finish(false));
890
+ sock.once("timeout", () => finish(false));
891
+ });
892
+ }
893
+ async function probeHttp(host, port, urlPath, headers, expectStatus) {
894
+ const ctrl = new AbortController();
895
+ const to = setTimeout(() => ctrl.abort(), 5000);
896
+ try {
897
+ const res = await fetch(`http://${host}:${port}${urlPath}`, {
898
+ signal: ctrl.signal,
899
+ headers,
900
+ });
901
+ return expectStatus !== undefined ? res.status === expectStatus : res.ok;
902
+ }
903
+ catch {
904
+ return false;
905
+ }
906
+ finally {
907
+ clearTimeout(to);
908
+ }
909
+ }
910
+ async function probeExec(name, command) {
911
+ const r = await docker(["exec", name, "sh", "-c", command], 10_000);
912
+ return r.code === 0;
913
+ }
914
+ async function waitForReady(svc) {
915
+ const check = svc.readyCheck;
916
+ if (!check)
917
+ return;
918
+ const timeoutSecs = check.timeoutSecs ?? 60;
919
+ const deadline = Date.now() + timeoutSecs * 1000;
920
+ // Ramped poll: a flat 500ms quantized every service's ready latency
921
+ // (and compounds down dependsOn chains). Fast early probes catch
922
+ // quick services; the ramp caps the polling load on slow ones. Exec
923
+ // probes keep a higher floor — each attempt spawns a docker exec.
924
+ const ramp = check.type === "exec" ? [250, 250, 400, 400, 500] : [50, 100, 150, 250, 400, 500];
925
+ let attempt = 0;
926
+ while (Date.now() < deadline) {
927
+ let ok = false;
928
+ if (check.type === "tcp") {
929
+ ok = await probeTcp(svc.name, check.port);
930
+ }
931
+ else if (check.type === "http") {
932
+ ok = await probeHttp(svc.name, check.port, check.path ?? "/", check.headers, check.expectStatus);
933
+ }
934
+ else {
935
+ ok = await probeExec(svc.name, check.command);
936
+ }
937
+ if (ok)
938
+ return;
939
+ const delay = ramp[Math.min(attempt, ramp.length - 1)];
940
+ attempt++;
941
+ await new Promise((r) => setTimeout(r, delay));
942
+ }
943
+ const logs = await docker(["logs", "--tail=200", svc.name], 30_000);
944
+ throw new Error(`service ${svc.name} not ready within ${timeoutSecs}s. Recent container logs:\n${logs.stdout}\n${logs.stderr}`);
945
+ }
946
+ /**
947
+ * Validate the `dependsOn` graph and return the name→service map used to
948
+ * walk it. Rejects unknown dependencies and cycles (the same two errors
949
+ * the old level scheduler raised) so the DAG runner can assume a clean
950
+ * graph.
951
+ */
952
+ function validateServiceGraph(services) {
953
+ const byName = new Map(services.map((s) => [s.name, s]));
954
+ for (const s of services) {
955
+ for (const d of s.dependsOn ?? []) {
956
+ if (!byName.has(d)) {
957
+ throw new Error(`service ${s.name} depends on unknown service ${d}`);
958
+ }
959
+ }
960
+ }
961
+ // Cycle detection via DFS coloring (white=unseen, gray=on stack, black=done).
962
+ const WHITE = 0, GRAY = 1, BLACK = 2;
963
+ const color = new Map(services.map((s) => [s.name, WHITE]));
964
+ const visit = (name) => {
965
+ color.set(name, GRAY);
966
+ for (const d of byName.get(name).dependsOn ?? []) {
967
+ const c = color.get(d);
968
+ if (c === GRAY)
969
+ throw new Error("service dependency cycle");
970
+ if (c === WHITE)
971
+ visit(d);
972
+ }
973
+ color.set(name, BLACK);
974
+ };
975
+ for (const s of services)
976
+ if (color.get(s.name) === WHITE)
977
+ visit(s.name);
978
+ return byName;
979
+ }
980
+ /**
981
+ * Bring up every service as early as its own dependencies allow.
982
+ *
983
+ * Each service starts the instant all of its `dependsOn` services have
984
+ * finished `startOne` (run → readyCheck → setup) — not when its whole
985
+ * topological "level" has. Independent branches run fully concurrently;
986
+ * a slow probe on one service delays only its own transitive dependents,
987
+ * never an unrelated branch. `startOne(svc)`'s promise is memoized so each
988
+ * service runs exactly once even when several dependents share a dep, and
989
+ * a dependency failure propagates by rejecting every dependent's await.
990
+ */
991
+ async function startServices(services, startOne) {
992
+ const byName = validateServiceGraph(services);
993
+ const started = new Map();
994
+ const start = (svc) => {
995
+ const existing = started.get(svc.name);
996
+ if (existing)
997
+ return existing;
998
+ const p = (async () => {
999
+ await Promise.all((svc.dependsOn ?? [])
1000
+ .filter((d) => byName.has(d))
1001
+ .map((d) => start(byName.get(d))));
1002
+ await startOne(svc);
1003
+ })();
1004
+ started.set(svc.name, p);
1005
+ return p;
1006
+ };
1007
+ await Promise.all(services.map(start));
1008
+ }
1009
+ // ────────────────────────────────────────────────────────────────────────
1010
+ // Ingress — in-daemon HTTP/HTTPS listeners that route by Host header.
1011
+ //
1012
+ // Two kinds of routes share the same listeners:
1013
+ //
1014
+ // * Fakes — in-daemon mock APIs. Each fake declares `hostnames` and a
1015
+ // `port` (default 80); the request hits the fake's handler with the
1016
+ // fake's `state`. HTTPS always serves on 443 (SNI per hostname,
1017
+ // leaf cert signed by the in-VM root CA).
1018
+ //
1019
+ // * Service TLS — reverse-proxy fronts for user services. Each
1020
+ // `services.<name>.tls` entry declares `{ hostname, port }`; the
1021
+ // daemon binds the hostname on :80 AND :443 and proxies each
1022
+ // request to `http://<service>:<port>` inside the docker network.
1023
+ // WebSocket upgrades are bridged. The leaf cert is signed by the
1024
+ // same root CA, so `ctx.browser()` and peer services trust it.
1025
+ //
1026
+ // Both listeners bind on 0.0.0.0 so containers reach them via the
1027
+ // bridge gateway IP (also written into /run/spectest-fakes.json for
1028
+ // spectest-resolver and injected as --add-host on every container).
1029
+ //
1030
+ // Per-fake `state` is plain JS memory and lives across snapshot/fork
1031
+ // along with the rest of the daemon — every fork sees its own copy.
1032
+ // ────────────────────────────────────────────────────────────────────────
1033
+ const FAKES_REGISTRY_PATH = process.env.SPECTEST_FAKES_REGISTRY ?? "/run/spectest-fakes.json";
1034
+ const DEFAULT_FAKE_PORT = 80;
1035
+ /** Fixed HTTPS port shared by every route (fakes + service-tls). */
1036
+ const INGRESS_HTTPS_PORT = 443;
1037
+ /** Fixed HTTP port always bound for service-tls (alongside any
1038
+ * fakes whose `port` happens to be 80). */
1039
+ const INGRESS_HTTP_PORT = 80;
1040
+ /** All loaded fakes, keyed by stable name (the `fakes` map key). Holds the
1041
+ * in-daemon handler, forked state, and helpers — the parts intrinsic to a
1042
+ * fake. Their *networking* (certs, DNS, routes) comes from `LOWERED`. */
1043
+ const FAKES = new Map();
1044
+ /** Generic ingress derived from the loaded project (tls/hostnames/fakes/
1045
+ * component `provides`) by the SDK's `lowerIngress`. The daemon executes
1046
+ * this and never reads `svc.tls`/`svc.hostnames` itself. Rebuilt on /load. */
1047
+ let LOWERED = {
1048
+ certificates: [],
1049
+ proxies: [],
1050
+ ingressHosts: [],
1051
+ aliasesByService: {},
1052
+ wildcards: [],
1053
+ };
1054
+ /** Running HTTP servers per port (Bun.Server). Rebuilt on /load. */
1055
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1056
+ const INGRESS_HTTP_SERVERS = new Map();
1057
+ /** Running HTTPS servers per port (currently always {INGRESS_HTTPS_PORT}). */
1058
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1059
+ const INGRESS_HTTPS_SERVERS = new Map();
1060
+ /**
1061
+ * Live per-port route tables, keyed by listen port (80 / 443 / fake ports).
1062
+ * Each listener's `fetch` closure captures *this* Map object, so adding an
1063
+ * entry takes effect immediately with no rebind — that's what lets a runtime
1064
+ * `tls` (a `ctx.startService({ tls })`) bind a new ingress route after boot.
1065
+ * Held at module scope so it's part of the live daemon process and forks
1066
+ * with the snapshot, exactly like fake state / the names REGISTRY. Rebuilt on
1067
+ * /load (cleared by {@link stopIngressServers}).
1068
+ */
1069
+ const INGRESS_ROUTES_BY_PORT = new Map();
1070
+ /**
1071
+ * The :443 SNI cert table: serverName → leaf. Unlike the route table, Bun's
1072
+ * TLS config is fixed at `Bun.serve` time (reload won't add an SNI entry), so
1073
+ * minting a cert for a *new* hostname requires rebinding the :443 listener
1074
+ * (cheap, ~1ms — see {@link rebindHttpsListener}). A hostname already covered
1075
+ * by an existing exact or wildcard cert needs no rebind, just a route entry.
1076
+ */
1077
+ const HTTPS_CERT_BY_HOST = new Map();
1078
+ /**
1079
+ * Tear down listener servers between /load calls so the new project's
1080
+ * routes can rebind cleanly.
1081
+ */
1082
+ function stopIngressServers() {
1083
+ for (const [port, srv] of INGRESS_HTTP_SERVERS) {
1084
+ try {
1085
+ srv.stop?.();
1086
+ }
1087
+ catch (err) {
1088
+ // eslint-disable-next-line no-console
1089
+ console.warn(`[ingress] failed to stop http server on :${port}:`, err);
1090
+ }
1091
+ }
1092
+ INGRESS_HTTP_SERVERS.clear();
1093
+ for (const [port, srv] of INGRESS_HTTPS_SERVERS) {
1094
+ try {
1095
+ srv.stop?.();
1096
+ }
1097
+ catch (err) {
1098
+ // eslint-disable-next-line no-console
1099
+ console.warn(`[ingress] failed to stop https server on :${port}:`, err);
1100
+ }
1101
+ }
1102
+ INGRESS_HTTPS_SERVERS.clear();
1103
+ INGRESS_ROUTES_BY_PORT.clear();
1104
+ HTTPS_CERT_BY_HOST.clear();
1105
+ }
1106
+ function buildIngress(project) {
1107
+ stopIngressServers();
1108
+ FAKES.clear();
1109
+ // Lower the friendly surface (tls/hostnames/provides/fakes) into the
1110
+ // generic decl set the daemon executes. The special-casing lives in the
1111
+ // SDK's lowerIngress, not here.
1112
+ LOWERED = lowerIngress(project);
1113
+ if (!project.fakes)
1114
+ return;
1115
+ for (const [name, def] of Object.entries(project.fakes)) {
1116
+ FAKES.set(name, {
1117
+ def,
1118
+ state: undefined, // built in startIngress after `state()` runs
1119
+ hostnames: def.hostnames.map((h) => h.toLowerCase()),
1120
+ port: def.port ?? DEFAULT_FAKE_PORT,
1121
+ });
1122
+ }
1123
+ }
1124
+ /**
1125
+ * Generate a leaf cert + key for one ingress route (fake or service
1126
+ * proxy), signed by the in-VM root CA at {CA_PATH}. SANs cover every
1127
+ * hostname the route answers to, so a client connecting with TLS
1128
+ * verifies cleanly regardless of which hostname it used. Shells out
1129
+ * to `openssl req -x509 -CA ... -CAkey ...` (OpenSSL 3.0+; Debian
1130
+ * bookworm ships 3.0.x).
1131
+ *
1132
+ * `label` is a short tag baked into the cert Subject CN and the temp
1133
+ * file names — only used for diagnostics, not for TLS verification.
1134
+ */
1135
+ async function generateHostCert(label, hostnames) {
1136
+ const id = `spectest-host-${sanitizeSegment(label)}-${randomUUID().slice(0, 8)}`;
1137
+ const keyPath = path.join("/tmp", `${id}.key`);
1138
+ const crtPath = path.join("/tmp", `${id}.crt`);
1139
+ const sans = hostnames.map((h) => `DNS:${h}`).join(",");
1140
+ const args = [
1141
+ "req",
1142
+ "-newkey",
1143
+ "rsa:2048",
1144
+ "-nodes",
1145
+ "-keyout",
1146
+ keyPath,
1147
+ "-out",
1148
+ crtPath,
1149
+ "-x509",
1150
+ "-CA",
1151
+ CA_PATH,
1152
+ "-CAkey",
1153
+ CA_KEY_PATH,
1154
+ "-days",
1155
+ "3650",
1156
+ "-subj",
1157
+ `/CN=spectest-${label}`,
1158
+ "-addext",
1159
+ `subjectAltName=${sans}`,
1160
+ "-addext",
1161
+ "basicConstraints=CA:FALSE",
1162
+ "-addext",
1163
+ "extendedKeyUsage=serverAuth",
1164
+ "-addext",
1165
+ "keyUsage=digitalSignature,keyEncipherment",
1166
+ ];
1167
+ const r = await shx("openssl", args, 30_000);
1168
+ if (r.code !== 0) {
1169
+ throw new Error(`openssl req for ${label} failed (rc=${r.code}): ${r.stderr.trim() || r.stdout.trim()}`);
1170
+ }
1171
+ try {
1172
+ const [cert, key] = await Promise.all([
1173
+ fs.readFile(crtPath, "utf8"),
1174
+ fs.readFile(keyPath, "utf8"),
1175
+ ]);
1176
+ return { cert, key };
1177
+ }
1178
+ finally {
1179
+ await Promise.all([
1180
+ fs.unlink(keyPath).catch(() => { }),
1181
+ fs.unlink(crtPath).catch(() => { }),
1182
+ ]);
1183
+ }
1184
+ }
1185
+ /** Resolve the spectest-net bridge gateway IP — the address containers
1186
+ * use to reach the VM host. Asks dockerd via the docker CLI; cached for
1187
+ * the daemon's life because the network is recreated only on reload. */
1188
+ let cachedGatewayIp = null;
1189
+ async function bridgeGatewayIp() {
1190
+ if (cachedGatewayIp)
1191
+ return cachedGatewayIp;
1192
+ const out = await docker([
1193
+ "network",
1194
+ "inspect",
1195
+ "--format",
1196
+ "{{(index .IPAM.Config 0).Gateway}}",
1197
+ NETWORK_NAME,
1198
+ ], 10_000);
1199
+ if (out.code !== 0) {
1200
+ throw new Error(`docker network inspect ${NETWORK_NAME} failed (rc=${out.code}): ${out.stderr.trim()}`);
1201
+ }
1202
+ const ip = out.stdout.trim();
1203
+ if (!ip)
1204
+ throw new Error(`no gateway IP returned for network ${NETWORK_NAME}`);
1205
+ cachedGatewayIp = ip;
1206
+ return ip;
1207
+ }
1208
+ /**
1209
+ * Pristine `fetch` captured at module load, before any test-scoped
1210
+ * fetch wrapper can monkey-patch `globalThis.fetch`. The reverse-proxy
1211
+ * uses this directly so its outbound HTTP calls aren't intercepted by
1212
+ * the test recorder — they'd be (a) misattributed to the test's
1213
+ * timeline, and (b) trip up the Response constructor because the
1214
+ * recorder wraps `res.status` / `res.body` in inspectable proxies
1215
+ * that don't pass through as primitives.
1216
+ */
1217
+ const NATIVE_FETCH = globalThis.fetch.bind(globalThis);
1218
+ /** Hop-by-hop headers per RFC 7230 §6.1 — never forwarded by a proxy. */
1219
+ const HOP_BY_HOP_HEADERS = new Set([
1220
+ "connection",
1221
+ "keep-alive",
1222
+ "proxy-authenticate",
1223
+ "proxy-authorization",
1224
+ "te",
1225
+ "trailers",
1226
+ "transfer-encoding",
1227
+ "upgrade",
1228
+ "host",
1229
+ ]);
1230
+ /**
1231
+ * Is this a CORS preflight? A preflight is the browser's own probe (never
1232
+ * app business logic): an `OPTIONS` carrying `Origin` +
1233
+ * `Access-Control-Request-Method`. Plain `OPTIONS` calls (no `ACRM`) are real
1234
+ * app requests and pass straight through to the upstream/fake.
1235
+ */
1236
+ function isCorsPreflight(req) {
1237
+ return (req.method === "OPTIONS" &&
1238
+ req.headers.has("origin") &&
1239
+ req.headers.has("access-control-request-method"));
1240
+ }
1241
+ /**
1242
+ * Answer a CORS preflight at the ingress, permissively, reflecting exactly
1243
+ * what the browser asked for.
1244
+ *
1245
+ * Why this belongs in the platform, not the app: inside the hermetic sandbox
1246
+ * the app page's origin (e.g. `http://<svc>.internal:<port>`) and every host
1247
+ * it fetches through this ingress (`https://api.example.com`) are *always*
1248
+ * different origins, so any request with a non-safelisted header — which
1249
+ * includes `Authorization`, and crucially `Cache-Control` / `Pragma` — is
1250
+ * preflighted by the browser. If we forward the `OPTIONS` to the upstream, the
1251
+ * request succeeds or fails on whether *that* app happens to enumerate the
1252
+ * header in its `Access-Control-Allow-Headers`. Real apps list `Authorization`
1253
+ * but almost never `Cache-Control`/`Pragma`, so a client that sends those (many
1254
+ * HTTP libraries add `Cache-Control: no-cache` by default) fails the preflight
1255
+ * with an instant "Failed to fetch" — even though the identical request works
1256
+ * in production behind a permissive edge/gateway. Reflecting
1257
+ * `Access-Control-Request-Headers` verbatim makes the ingress transparent to
1258
+ * whatever header vocabulary the app under test uses.
1259
+ */
1260
+ function corsPreflightResponse(req) {
1261
+ const origin = req.headers.get("origin") ?? "*";
1262
+ const reqHeaders = req.headers.get("access-control-request-headers");
1263
+ const reqMethod = req.headers.get("access-control-request-method");
1264
+ const headers = new Headers();
1265
+ headers.set("access-control-allow-origin", origin);
1266
+ // Echo the specific origin (not `*`) so credentialed requests are allowed;
1267
+ // `Allow-Origin: *` + `Allow-Credentials: true` is a spec violation browsers
1268
+ // reject.
1269
+ headers.set("access-control-allow-credentials", "true");
1270
+ headers.set("access-control-allow-methods", reqMethod && reqMethod.length > 0
1271
+ ? reqMethod
1272
+ : "GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS");
1273
+ headers.set("access-control-allow-headers", reqHeaders && reqHeaders.length > 0 ? reqHeaders : "*");
1274
+ headers.set("access-control-max-age", "600");
1275
+ // The response varies by the reflected origin/headers — keep caches honest.
1276
+ headers.append("vary", "Origin");
1277
+ headers.append("vary", "Access-Control-Request-Headers");
1278
+ return new Response(null, { status: 204, headers });
1279
+ }
1280
+ /**
1281
+ * Make sure the browser sees an `Access-Control-Allow-Origin` it accepts on the
1282
+ * *actual* cross-origin response. Only fills one in when the upstream/fake
1283
+ * didn't set its own, so an app that manages CORS itself keeps full control;
1284
+ * this just stops a missing header from turning an otherwise-fine 200 into a
1285
+ * "Failed to fetch". No-op for same-origin requests (no `Origin`).
1286
+ */
1287
+ function augmentCorsResponse(req, res) {
1288
+ const origin = req.headers.get("origin");
1289
+ if (!origin)
1290
+ return res;
1291
+ if (res.headers.has("access-control-allow-origin"))
1292
+ return res;
1293
+ try {
1294
+ res.headers.set("access-control-allow-origin", origin);
1295
+ res.headers.set("access-control-allow-credentials", "true");
1296
+ res.headers.append("vary", "Origin");
1297
+ }
1298
+ catch {
1299
+ // Some responses (e.g. a 101 upgrade stub) carry guarded/immutable
1300
+ // headers — leave those untouched.
1301
+ }
1302
+ return res;
1303
+ }
1304
+ /**
1305
+ * Bring ingress servers up: bind one Bun.serve per unique HTTP port
1306
+ * (fakes' ports plus the always-on :80 for service proxies), plus a
1307
+ * shared HTTPS :443 (SNI per hostname). Build each fake's initial state,
1308
+ * then write the hostname→ip registry that spectest-resolver consults
1309
+ * for DNS. Idempotent — calling twice rebuilds.
1310
+ */
1311
+ async function startIngress() {
1312
+ const hasIngress = FAKES.size > 0 ||
1313
+ LOWERED.proxies.length > 0 ||
1314
+ LOWERED.ingressHosts.length > 0 ||
1315
+ LOWERED.wildcards.length > 0;
1316
+ if (!hasIngress) {
1317
+ // Make sure the resolver doesn't see stale entries from a prior project.
1318
+ REGISTRY.hosts = {};
1319
+ REGISTRY.wildcards = [];
1320
+ await writeRegistry();
1321
+ return;
1322
+ }
1323
+ const gw = await bridgeGatewayIp();
1324
+ // Initialise per-fake state. Awaited sequentially — state factories
1325
+ // are expected to be tiny constructors; the cost of serial init is
1326
+ // dwarfed by the eventual snapshot.
1327
+ for (const [name, fake] of FAKES) {
1328
+ if (fake.def.state) {
1329
+ try {
1330
+ fake.state = await fake.def.state();
1331
+ }
1332
+ catch (err) {
1333
+ throw new Error(`fake ${JSON.stringify(name)} state() factory threw: ${err.message}`);
1334
+ }
1335
+ }
1336
+ else {
1337
+ fake.state = {};
1338
+ }
1339
+ }
1340
+ // Mint one leaf cert per `certificate` decl (SANs = its hostnames),
1341
+ // signed by the in-VM root CA, and index it by hostname for SNI. Done
1342
+ // before binding so the HTTPS listener has certs ready and a startup
1343
+ // failure aborts /bootstrap cleanly.
1344
+ const caPresent = existsSync(CA_PATH) && existsSync(CA_KEY_PATH);
1345
+ if (caPresent) {
1346
+ for (const group of LOWERED.certificates) {
1347
+ if (group.hostnames.length === 0)
1348
+ continue;
1349
+ const leaf = await generateHostCert(group.hostnames[0], group.hostnames);
1350
+ for (const h of group.hostnames)
1351
+ HTTPS_CERT_BY_HOST.set(h, leaf);
1352
+ }
1353
+ }
1354
+ else if (LOWERED.certificates.length > 0) {
1355
+ // eslint-disable-next-line no-console
1356
+ console.warn(`[ingress] root CA missing at ${CA_PATH}; skipping HTTPS bind (fakes/proxies will be HTTP-only)`);
1357
+ }
1358
+ const Bun = requireBun();
1359
+ // Resolve every ingress hostname to its handler into the live per-port
1360
+ // route tables (module scope, so runtime `tls` can extend them later).
1361
+ // Fakes run an in-daemon handler on their declared port; proxies
1362
+ // reverse-proxy to a service:port and bind :80 (HTTPS, if any, is :443).
1363
+ const ensurePort = (port) => {
1364
+ const m = INGRESS_ROUTES_BY_PORT.get(port) ?? new Map();
1365
+ INGRESS_ROUTES_BY_PORT.set(port, m);
1366
+ return m;
1367
+ };
1368
+ for (const fake of FAKES.values()) {
1369
+ if (fake.port === INGRESS_HTTPS_PORT)
1370
+ continue;
1371
+ const routes = ensurePort(fake.port);
1372
+ for (const h of fake.hostnames)
1373
+ routes.set(h, { kind: "fake", fake });
1374
+ }
1375
+ if (LOWERED.proxies.length > 0) {
1376
+ const routes = ensurePort(INGRESS_HTTP_PORT);
1377
+ for (const p of LOWERED.proxies) {
1378
+ routes.set(p.hostname, { kind: "proxy", service: p.service, port: p.port });
1379
+ }
1380
+ }
1381
+ // The :443 route table mirrors every certificated hostname's handler.
1382
+ if (HTTPS_CERT_BY_HOST.size > 0) {
1383
+ const httpsRoutes = ensurePort(INGRESS_HTTPS_PORT);
1384
+ for (const h of HTTPS_CERT_BY_HOST.keys()) {
1385
+ for (const fake of FAKES.values()) {
1386
+ if (fake.hostnames.includes(h))
1387
+ httpsRoutes.set(h, { kind: "fake", fake });
1388
+ }
1389
+ const proxy = LOWERED.proxies.find((p) => p.hostname === h);
1390
+ if (proxy)
1391
+ httpsRoutes.set(h, { kind: "proxy", service: proxy.service, port: proxy.port });
1392
+ }
1393
+ }
1394
+ // ── HTTP listeners (one per non-443 port).
1395
+ for (const [port, byHost] of INGRESS_ROUTES_BY_PORT) {
1396
+ if (port === INGRESS_HTTPS_PORT)
1397
+ continue;
1398
+ INGRESS_HTTP_SERVERS.set(port, bindIngressServer(Bun, port, byHost, `port ${port}`));
1399
+ // eslint-disable-next-line no-console
1400
+ console.log(`[ingress] http :${port} for ${[...byHost.keys()].join(", ")}`);
1401
+ }
1402
+ // ── HTTPS listener on INGRESS_HTTPS_PORT: SNI per certificated hostname.
1403
+ if (HTTPS_CERT_BY_HOST.size > 0)
1404
+ rebindHttpsListener(Bun);
1405
+ // Seed the resolver's names registry: ingress hostnames (fakes, TLS
1406
+ // proxies, dnsName(→ingress)) → bridge gateway, plus ingress-targeted
1407
+ // wildcards. Service-targeted wildcards wait for the post-container pass
1408
+ // (their containers aren't up yet). Dynamic ctx.dnsName calls extend this.
1409
+ await seedNamesRegistry({ servicesUp: false });
1410
+ }
1411
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1412
+ function requireBun() {
1413
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1414
+ const Bun = globalThis.Bun;
1415
+ if (!Bun?.serve) {
1416
+ throw new Error("ingress requires Bun.serve; the daemon must run under Bun (it does in-VM)");
1417
+ }
1418
+ return Bun;
1419
+ }
1420
+ /** Flatten {@link HTTPS_CERT_BY_HOST} into Bun's TLS-entry SNI array. */
1421
+ function tlsEntriesFromCerts() {
1422
+ return [...HTTPS_CERT_BY_HOST].map(([serverName, leaf]) => ({
1423
+ cert: leaf.cert,
1424
+ key: leaf.key,
1425
+ serverName,
1426
+ }));
1427
+ }
1428
+ /**
1429
+ * (Re)bind the :443 listener from the current cert table + route map. Bun's
1430
+ * TLS config is immutable per `Bun.serve`, so adding an SNI cert means
1431
+ * stopping the old listener and serving a fresh one — cheap (~1ms) and the
1432
+ * window is sub-millisecond. The route Map is the persistent module object,
1433
+ * so the new listener closes over the same table (later route additions need
1434
+ * no rebind). No-ops to a plain rebind when only routes changed.
1435
+ */
1436
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1437
+ function rebindHttpsListener(Bun) {
1438
+ const routes = INGRESS_ROUTES_BY_PORT.get(INGRESS_HTTPS_PORT) ?? new Map();
1439
+ INGRESS_ROUTES_BY_PORT.set(INGRESS_HTTPS_PORT, routes);
1440
+ const old = INGRESS_HTTPS_SERVERS.get(INGRESS_HTTPS_PORT);
1441
+ if (old) {
1442
+ try {
1443
+ old.stop(true);
1444
+ }
1445
+ catch (err) {
1446
+ // eslint-disable-next-line no-console
1447
+ console.warn("[ingress] failed to stop https listener for rebind:", err);
1448
+ }
1449
+ }
1450
+ const server = bindIngressServer(Bun, INGRESS_HTTPS_PORT, routes, `https :${INGRESS_HTTPS_PORT}`, tlsEntriesFromCerts());
1451
+ INGRESS_HTTPS_SERVERS.set(INGRESS_HTTPS_PORT, server);
1452
+ // eslint-disable-next-line no-console
1453
+ console.log(`[ingress] https :${INGRESS_HTTPS_PORT} for ${[...routes.keys()].join(", ")}`);
1454
+ }
1455
+ /** True if an exact or wildcard cert already covers `hostname` for SNI. */
1456
+ function certCovers(hostname) {
1457
+ if (HTTPS_CERT_BY_HOST.has(hostname))
1458
+ return true;
1459
+ for (const serverName of HTTPS_CERT_BY_HOST.keys()) {
1460
+ if (isWildcard(serverName) && hostname.endsWith(wildcardSuffix(serverName))) {
1461
+ return true;
1462
+ }
1463
+ }
1464
+ return false;
1465
+ }
1466
+ /**
1467
+ * Bind a runtime ingress route for one `tls: [{ hostname, port }]` entry on a
1468
+ * {@link RuntimeServiceSpec} — the runtime twin of a boot service's `tls`.
1469
+ * Mints a leaf cert (unless one already covers the hostname), stands up a
1470
+ * TLS-terminating reverse proxy at `https://<hostname>/` → `service:port`,
1471
+ * also serves plain `http://<hostname>/`, and points the hostname at the
1472
+ * daemon gateway in the resolver registry. Idempotent per hostname.
1473
+ *
1474
+ * Everything it mutates (the live route tables, the :443 cert table, the
1475
+ * names REGISTRY) is daemon-process state, so the binding forks with the
1476
+ * per-test snapshot exactly like fake state — a `dependsOn` child inherits
1477
+ * it, siblings forked from an earlier snapshot never see it.
1478
+ */
1479
+ async function bindRuntimeTls(hostname, service, port) {
1480
+ // Reuse the boot primitives for validation + lowercasing; throws on a
1481
+ // malformed hostname / upstream just like a boot `tls` would at load.
1482
+ const decl = makeProxyDecl(hostname, { service, port });
1483
+ const host = decl.hostname;
1484
+ if (!existsSync(CA_PATH) || !existsSync(CA_KEY_PATH)) {
1485
+ throw new Error(`runtime tls for ${JSON.stringify(host)} requires the in-VM root CA at ${CA_PATH}`);
1486
+ }
1487
+ const Bun = requireBun();
1488
+ const route = { kind: "proxy", service, port };
1489
+ // Plain HTTP on :80 (parity with boot `tls`, which serves both schemes).
1490
+ let httpRoutes = INGRESS_ROUTES_BY_PORT.get(INGRESS_HTTP_PORT);
1491
+ if (!httpRoutes) {
1492
+ httpRoutes = new Map();
1493
+ INGRESS_ROUTES_BY_PORT.set(INGRESS_HTTP_PORT, httpRoutes);
1494
+ }
1495
+ httpRoutes.set(host, route);
1496
+ if (!INGRESS_HTTP_SERVERS.has(INGRESS_HTTP_PORT)) {
1497
+ INGRESS_HTTP_SERVERS.set(INGRESS_HTTP_PORT, bindIngressServer(Bun, INGRESS_HTTP_PORT, httpRoutes, `port ${INGRESS_HTTP_PORT}`));
1498
+ }
1499
+ // HTTPS on :443. A new cert forces a listener rebind; an already-covered
1500
+ // hostname (exact dup or a boot wildcard) just needs the route entry.
1501
+ const needCert = !certCovers(host);
1502
+ if (needCert) {
1503
+ HTTPS_CERT_BY_HOST.set(host, await generateHostCert(host, [host]));
1504
+ }
1505
+ const httpsRoutes = INGRESS_ROUTES_BY_PORT.get(INGRESS_HTTPS_PORT) ?? new Map();
1506
+ INGRESS_ROUTES_BY_PORT.set(INGRESS_HTTPS_PORT, httpsRoutes);
1507
+ httpsRoutes.set(host, route);
1508
+ if (needCert || !INGRESS_HTTPS_SERVERS.has(INGRESS_HTTPS_PORT)) {
1509
+ rebindHttpsListener(Bun);
1510
+ }
1511
+ // Resolve the hostname to the daemon gateway (where :443/:80 listen).
1512
+ const gw = await bridgeGatewayIp();
1513
+ REGISTRY.hosts[host] = gw;
1514
+ await writeRegistry();
1515
+ // eslint-disable-next-line no-console
1516
+ console.log(`[ingress] runtime https ${host} -> ${service}:${port}`);
1517
+ }
1518
+ /**
1519
+ * Undo {@link bindRuntimeTls} for one hostname when its runtime service is
1520
+ * stopped: drop the route (so it 404s) and the registry entry. The cert is
1521
+ * left in the SNI table — harmless without a route, and removing it would
1522
+ * mean an avoidable :443 rebind.
1523
+ */
1524
+ async function unbindRuntimeTls(hostname) {
1525
+ const host = hostname.toLowerCase();
1526
+ INGRESS_ROUTES_BY_PORT.get(INGRESS_HTTP_PORT)?.delete(host);
1527
+ INGRESS_ROUTES_BY_PORT.get(INGRESS_HTTPS_PORT)?.delete(host);
1528
+ if (host in REGISTRY.hosts) {
1529
+ delete REGISTRY.hosts[host];
1530
+ await writeRegistry();
1531
+ }
1532
+ }
1533
+ /**
1534
+ * Spin up one Bun.serve listener bound to (port, optional TLS) that
1535
+ * dispatches every request to the matching Route by Host header.
1536
+ *
1537
+ * Shared by the HTTP and HTTPS branches. Also exports a `websocket`
1538
+ * handler so reverse-proxy targets can transparently bridge WS
1539
+ * upgrades through to their upstream service.
1540
+ */
1541
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1542
+ function bindIngressServer(
1543
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1544
+ Bun, port, byHost, listenerLabel, tlsEntries) {
1545
+ // A TLS listener terminates https; everything else is plain http. Used
1546
+ // to stamp X-Forwarded-Proto so upstreams that build absolute URLs or
1547
+ // redirect see the scheme the client actually used, not our http hop.
1548
+ const proto = tlsEntries ? "https" : "http";
1549
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1550
+ const opts = {
1551
+ port,
1552
+ hostname: "0.0.0.0",
1553
+ // Bun.serve defaults to a 10s idleTimeout, which kills any proxied
1554
+ // request whose upstream takes >10s to produce bytes — under parallel
1555
+ // test load that surfaced as "fetch failed"/"other side closed" on
1556
+ // deploy-archive uploads and ERR_EMPTY_RESPONSE in browser tests
1557
+ // ([Bun.serve]: request timed out after 10 seconds). Ingress fronts
1558
+ // arbitrarily slow app endpoints (deploys can legitimately take
1559
+ // minutes), so disable the idle timeout entirely; forked test VMs are
1560
+ // short-lived, leaked-connection risk is bounded by the fork.
1561
+ idleTimeout: 0,
1562
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1563
+ fetch: (req, server) => dispatchIngress(req, server, byHost, listenerLabel, proto),
1564
+ websocket: {
1565
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1566
+ async open(ws) {
1567
+ const data = ws.data;
1568
+ try {
1569
+ const upstream = new WebSocket(data.upstreamUrl);
1570
+ // ArrayBuffer so binary frames can be ws.send()'d to the
1571
+ // downstream client verbatim — Blob would need an extra
1572
+ // .arrayBuffer() round-trip on every message.
1573
+ upstream.binaryType = "arraybuffer";
1574
+ data.upstream = upstream;
1575
+ upstream.addEventListener("open", () => {
1576
+ for (const m of data.pending)
1577
+ upstream.send(m);
1578
+ data.pending = [];
1579
+ });
1580
+ upstream.addEventListener("message", (ev) => {
1581
+ try {
1582
+ ws.send(ev.data);
1583
+ }
1584
+ catch {
1585
+ /* client gone */
1586
+ }
1587
+ });
1588
+ upstream.addEventListener("close", (ev) => {
1589
+ try {
1590
+ ws.close(ev.code, ev.reason);
1591
+ }
1592
+ catch {
1593
+ /* already closed */
1594
+ }
1595
+ });
1596
+ upstream.addEventListener("error", () => {
1597
+ try {
1598
+ ws.close(1011, "upstream error");
1599
+ }
1600
+ catch {
1601
+ /* already closed */
1602
+ }
1603
+ });
1604
+ }
1605
+ catch (err) {
1606
+ // eslint-disable-next-line no-console
1607
+ console.warn(`[ingress] ws upstream open failed for ${data.upstreamUrl}:`, err);
1608
+ try {
1609
+ ws.close(1011, "upstream open failed");
1610
+ }
1611
+ catch {
1612
+ /* ignore */
1613
+ }
1614
+ }
1615
+ },
1616
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1617
+ message(ws, message) {
1618
+ const data = ws.data;
1619
+ const payload = typeof message === "string" ? message : new Uint8Array(message);
1620
+ if (data.upstream && data.upstream.readyState === WebSocket.OPEN) {
1621
+ data.upstream.send(payload);
1622
+ }
1623
+ else {
1624
+ // Buffer until the upstream finishes its handshake.
1625
+ data.pending.push(payload);
1626
+ }
1627
+ },
1628
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1629
+ close(ws, code, reason) {
1630
+ const data = ws.data;
1631
+ try {
1632
+ data.upstream?.close(code, reason);
1633
+ }
1634
+ catch {
1635
+ /* ignore */
1636
+ }
1637
+ },
1638
+ },
1639
+ };
1640
+ if (tlsEntries)
1641
+ opts.tls = tlsEntries;
1642
+ return Bun.serve(opts);
1643
+ }
1644
+ /**
1645
+ * Per-request dispatch shared by every ingress listener. Looks up the
1646
+ * Route by Host header (port stripped) and either:
1647
+ * - fake: invokes the handler, wraps thrown errors as 500;
1648
+ * - proxy: WebSocket upgrade → server.upgrade(); else reverse-proxy
1649
+ * to the upstream service over plain HTTP.
1650
+ */
1651
+ async function dispatchIngress(req,
1652
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1653
+ server, byHost, listenerLabel, proto) {
1654
+ const host = (req.headers.get("host") ?? "")
1655
+ .toLowerCase()
1656
+ .split(":")[0]
1657
+ .trim();
1658
+ const route = byHost.get(host);
1659
+ if (!route) {
1660
+ return new Response(`spectest-daemon: no ingress route bound to Host=${JSON.stringify(host)} on ${listenerLabel}\n`, { status: 404, headers: { "content-type": "text/plain" } });
1661
+ }
1662
+ // Answer CORS preflights at the ingress (see corsPreflightResponse) so a
1663
+ // cross-origin browser request carrying any header — Authorization,
1664
+ // Cache-Control, Pragma, … — isn't rejected by whatever the upstream happens
1665
+ // to list in Access-Control-Allow-Headers.
1666
+ if (isCorsPreflight(req))
1667
+ return corsPreflightResponse(req);
1668
+ if (route.kind === "fake") {
1669
+ try {
1670
+ const res = await route.fake.def.handler(req, route.fake.state, FAKE_CTX);
1671
+ return augmentCorsResponse(req, res);
1672
+ }
1673
+ catch (err) {
1674
+ const e = err;
1675
+ return new Response(`spectest-daemon: fake ${route.fake.def.name} threw: ${e?.message ?? String(err)}\n`, { status: 500, headers: { "content-type": "text/plain" } });
1676
+ }
1677
+ }
1678
+ const res = await proxyToService(req, server, route.service, route.port, listenerLabel, proto);
1679
+ return augmentCorsResponse(req, res);
1680
+ }
1681
+ /**
1682
+ * Reverse-proxy a request to `http://<service>:<port>` on
1683
+ * `spectest-net`. Handles plain HTTP/1.1 + 2 and WebSocket upgrades:
1684
+ *
1685
+ * - WS upgrade requests get routed through `server.upgrade()`, with
1686
+ * the upstream URL stashed on `ws.data`. The shared `websocket`
1687
+ * handler opens the upstream and bridges frames both ways.
1688
+ * - Plain requests pass through via `fetch()` with hop-by-hop
1689
+ * headers stripped; the response body is a ReadableStream returned
1690
+ * verbatim, so it streams back without buffering.
1691
+ *
1692
+ * `decompress: false` makes this a true byte-for-byte pass-through:
1693
+ * Bun's fetch otherwise auto-decompresses the upstream body, which would
1694
+ * leave us forwarding the original `Content-Encoding`/`Content-Length`
1695
+ * over a now-plaintext body — browsers then fail with
1696
+ * ERR_CONTENT_DECODING_FAILED or truncate on the stale length (the
1697
+ * WHATWG fetch footgun in whatwg/fetch#1729). Keeping the body encoded
1698
+ * means those headers still describe the bytes we send, and we relay the
1699
+ * client's `Accept-Encoding` upstream so the upstream picks the scheme.
1700
+ */
1701
+ async function proxyToService(req,
1702
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1703
+ server, service, port, listenerLabel, proto) {
1704
+ const url = new URL(req.url);
1705
+ const upstreamPath = `${url.pathname}${url.search}`;
1706
+ const upgrade = req.headers.get("upgrade")?.toLowerCase() ?? "";
1707
+ if (upgrade === "websocket") {
1708
+ const upstreamUrl = `ws://${await proxyUpstreamHost(service)}:${port}${upstreamPath}`;
1709
+ const wsData = {
1710
+ upstreamUrl,
1711
+ upstream: null,
1712
+ pending: [],
1713
+ };
1714
+ const ok = server.upgrade(req, { data: wsData });
1715
+ if (ok) {
1716
+ // Bun has already taken over the response — return a stub.
1717
+ return new Response(null, { status: 101 });
1718
+ }
1719
+ return new Response(`spectest-daemon: ws upgrade refused on ${listenerLabel}\n`, { status: 426, headers: { "content-type": "text/plain" } });
1720
+ }
1721
+ const fwdHeaders = new Headers();
1722
+ for (const [k, v] of req.headers) {
1723
+ if (HOP_BY_HOP_HEADERS.has(k.toLowerCase()))
1724
+ continue;
1725
+ fwdHeaders.append(k, v);
1726
+ }
1727
+ // Standard reverse-proxy provenance headers: the upstream sees the
1728
+ // public scheme/host it was reached through and the client's address,
1729
+ // even though we rewrite Host below to the service-net name.
1730
+ const clientIp = server.requestIP?.(req)?.address;
1731
+ const priorXff = req.headers.get("x-forwarded-for");
1732
+ const xff = clientIp ? (priorXff ? `${priorXff}, ${clientIp}` : clientIp) : priorXff;
1733
+ if (xff)
1734
+ fwdHeaders.set("x-forwarded-for", xff);
1735
+ fwdHeaders.set("x-forwarded-proto", proto);
1736
+ const publicHost = req.headers.get("host");
1737
+ if (publicHost)
1738
+ fwdHeaders.set("x-forwarded-host", publicHost);
1739
+ // Override Host so the upstream sees its own service-net name, not
1740
+ // the public hostname. Lets origin servers that vhost by Host header
1741
+ // continue to find the right virtual host.
1742
+ fwdHeaders.set("host", `${service}:${port}`);
1743
+ // Fresh connection per upstream request — never reuse a pooled
1744
+ // keep-alive conn. Upstreams with short idle timeouts (uvicorn defaults
1745
+ // to 5s) close pooled connections under Bun's fetch, and the next
1746
+ // request on the dead socket fails with "socket closed unexpectedly"
1747
+ // even though the service is healthy. In-VM connects to a peer
1748
+ // container are sub-ms, so per-request connects cost nothing at test
1749
+ // scale. `connection: close` makes the upstream tear down immediately;
1750
+ // `keepalive: false` on the fetch below keeps Bun from pooling its end.
1751
+ // (Both verified effective on Bun 1.3.14.)
1752
+ fwdHeaders.set("connection", "close");
1753
+ // Buffer bounded request bodies so a transient upstream connect failure
1754
+ // can be retried (a ReadableStream body is consumed by the first
1755
+ // attempt). Under heavy parallel-fork load an in-guest connect to a
1756
+ // peer container occasionally fails outright ("Unable to connect" on a
1757
+ // healthy upstream) — observed on deploy-tarball uploads to s3mock; a
1758
+ // bounded retry absorbs it. Bodies above the cap (or with unknown
1759
+ // length and a stream that exceeds it) keep streaming semantics and
1760
+ // simply don't retry.
1761
+ const RETRY_BODY_CAP = 128 * 1024 * 1024;
1762
+ const hasBody = req.method !== "GET" && req.method !== "HEAD";
1763
+ // Only bodies with a known, bounded length are buffered — an unknown
1764
+ // (chunked/streaming) length could be an endless client stream, which
1765
+ // must keep flowing through, not accumulate.
1766
+ const declaredLen = Number(req.headers.get("content-length") ?? NaN);
1767
+ let bufferedBody;
1768
+ if (hasBody && Number.isFinite(declaredLen) && declaredLen <= RETRY_BODY_CAP) {
1769
+ try {
1770
+ bufferedBody = await req.arrayBuffer();
1771
+ }
1772
+ catch {
1773
+ /* client aborted mid-upload; fall through, attempt will fail */
1774
+ }
1775
+ }
1776
+ const retryable = !hasBody || (bufferedBody !== undefined && bufferedBody.byteLength <= RETRY_BODY_CAP);
1777
+ const attempts = retryable ? 3 : 1;
1778
+ let lastErr;
1779
+ for (let attempt = 1; attempt <= attempts; attempt++) {
1780
+ // Resolve the upstream per attempt: a connect failure below drops the
1781
+ // cached IP, so a retry re-inspects the container.
1782
+ const upstreamUrl = `http://${await proxyUpstreamHost(service)}:${port}${upstreamPath}`;
1783
+ try {
1784
+ const upstreamReq = new Request(upstreamUrl, {
1785
+ method: req.method,
1786
+ headers: fwdHeaders,
1787
+ body: hasBody ? (bufferedBody ?? req.body) : undefined,
1788
+ redirect: "manual",
1789
+ });
1790
+ // decompress:false → forward the encoded body untouched (see fn doc).
1791
+ // keepalive:false → fresh connection per request (see fwdHeaders above).
1792
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1793
+ const upstreamRes = await NATIVE_FETCH(upstreamReq, {
1794
+ decompress: false,
1795
+ keepalive: false,
1796
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
1797
+ });
1798
+ // Strip hop-by-hop response headers; let Bun set content-length / TE.
1799
+ const respHeaders = new Headers();
1800
+ for (const [k, v] of upstreamRes.headers) {
1801
+ if (HOP_BY_HOP_HEADERS.has(k.toLowerCase()))
1802
+ continue;
1803
+ respHeaders.append(k, v);
1804
+ }
1805
+ return new Response(upstreamRes.body, {
1806
+ status: upstreamRes.status,
1807
+ statusText: upstreamRes.statusText,
1808
+ headers: respHeaders,
1809
+ });
1810
+ }
1811
+ catch (err) {
1812
+ lastErr = err;
1813
+ // Retry connection-level failures: connect errors, plus a socket
1814
+ // that died before any response bytes ("socket closed unexpectedly",
1815
+ // ECONNRESET, hang-up) — under boot-time bursts an upstream accepts
1816
+ // and drops connections while still warming up. Pre-response
1817
+ // failures are the standard retry class for reverse proxies
1818
+ // (nginx's proxy_next_upstream error). Anything that produced a
1819
+ // response is never replayed.
1820
+ const msg = err?.message ?? String(err);
1821
+ const connectFailure = /unable to connect|connection refused|connect|typo in the url|socket closed|connection closed|econnreset|socket hang ?up|epipe/i.test(msg);
1822
+ if (!connectFailure || attempt === attempts)
1823
+ break;
1824
+ // The cached IP may be stale (container recreated) — re-resolve.
1825
+ PROXY_IP_CACHE.delete(service);
1826
+ // eslint-disable-next-line no-console
1827
+ console.warn(`[ingress] upstream ${service}:${port} connect failed (attempt ${attempt}/${attempts}), retrying: ${msg}`);
1828
+ await new Promise((r) => setTimeout(r, 250 * attempt));
1829
+ }
1830
+ }
1831
+ const e = lastErr;
1832
+ return new Response(`spectest-daemon: upstream ${service}:${port} unreachable: ${e?.message ?? String(lastErr)}\n`, { status: 502, headers: { "content-type": "text/plain" } });
1833
+ }
1834
+ /**
1835
+ * In-memory names registry, serialised to FAKES_REGISTRY_PATH for the
1836
+ * resolver. `startIngress` seeds it from LOWERED (the static
1837
+ * tls/hostnames/fakes/wildcard decls); `registerDnsName` mutates it live
1838
+ * when a test calls `ctx.dnsName`. It lives in daemon memory, so it forks
1839
+ * with the rest of the snapshot — a test's dynamic registration is
1840
+ * isolated to its own fork, exactly like fake state.
1841
+ */
1842
+ const REGISTRY = { hosts: {}, wildcards: [] };
1843
+ async function writeRegistry() {
1844
+ const body = JSON.stringify({
1845
+ hosts: REGISTRY.hosts,
1846
+ wildcards: REGISTRY.wildcards,
1847
+ updatedAt: Date.now(),
1848
+ });
1849
+ try {
1850
+ await fs.mkdir(path.dirname(FAKES_REGISTRY_PATH), { recursive: true });
1851
+ await fs.writeFile(FAKES_REGISTRY_PATH, body);
1852
+ }
1853
+ catch (err) {
1854
+ // Resolver gracefully degrades; just log.
1855
+ // eslint-disable-next-line no-console
1856
+ console.warn(`[names] failed to write registry at ${FAKES_REGISTRY_PATH}:`, err);
1857
+ }
1858
+ }
1859
+ /** `*.example.com` → `.example.com` — the suffix the resolver matches. */
1860
+ function wildcardSuffix(pattern) {
1861
+ return pattern.slice(1); // drop the leading "*"
1862
+ }
1863
+ /** A service container's IP on spectest-net. `null` if the container isn't
1864
+ * up or isn't attached to the network yet. */
1865
+ async function serviceContainerIp(name) {
1866
+ const out = await docker([
1867
+ "inspect",
1868
+ "--format",
1869
+ `{{(index .NetworkSettings.Networks "${NETWORK_NAME}").IPAddress}}`,
1870
+ name,
1871
+ ], 10_000);
1872
+ if (out.code !== 0)
1873
+ return null;
1874
+ const ip = out.stdout.trim();
1875
+ return ip.length > 0 && ip !== "<no value>" ? ip : null;
1876
+ }
1877
+ /** Cache of service container IPs for the ingress proxy, so the proxy hot
1878
+ * path doesn't depend on in-guest DNS: name resolution goes through
1879
+ * spectest-resolver — a single-threaded Bun process that can be starved
1880
+ * when the guest's vCPUs are saturated (observed as ~30s of
1881
+ * "tarballs-s3:9090 unreachable" 502s during parallel deploy tests while
1882
+ * the container was healthy). Populated lazily via `docker inspect`
1883
+ * (local socket, no DNS); the proxy drops an entry on connect failure so
1884
+ * a recreated container re-resolves on retry. */
1885
+ const PROXY_IP_CACHE = new Map();
1886
+ async function proxyUpstreamHost(service) {
1887
+ const cached = PROXY_IP_CACHE.get(service);
1888
+ if (cached)
1889
+ return cached;
1890
+ const ip = await serviceContainerIp(service).catch(() => null);
1891
+ if (ip) {
1892
+ PROXY_IP_CACHE.set(service, ip);
1893
+ return ip;
1894
+ }
1895
+ // Fall back to the name (resolver / docker DNS) — e.g. a target that
1896
+ // isn't a docker container on spectest-net.
1897
+ return service;
1898
+ }
1899
+ /** Resolve a DnsTarget to a concrete IP: ingress → bridge gateway, service
1900
+ * → that container's IP. Throws if a service target has no IP yet. */
1901
+ async function resolveDnsTarget(target) {
1902
+ if ("ingress" in target)
1903
+ return bridgeGatewayIp();
1904
+ const ip = await serviceContainerIp(target.service);
1905
+ if (!ip) {
1906
+ throw new Error(`dnsName target service ${JSON.stringify(target.service)} has no IP on ${NETWORK_NAME} (is it a running service?)`);
1907
+ }
1908
+ return ip;
1909
+ }
1910
+ /**
1911
+ * Seed REGISTRY from the static lowered decls and write it. Run twice per
1912
+ * /bootstrap: once from startIngress (`servicesUp: false`) so ingress
1913
+ * hostnames answer during container startup, then once after every
1914
+ * container is up (`servicesUp: true`) so service-targeted wildcards (e.g.
1915
+ * k3s `ingressDomains`) can resolve their container IPs. The post-container
1916
+ * pass is what's captured into the warm template.
1917
+ */
1918
+ async function seedNamesRegistry(opts) {
1919
+ const gw = await bridgeGatewayIp();
1920
+ REGISTRY.hosts = {};
1921
+ REGISTRY.wildcards = [];
1922
+ for (const h of LOWERED.ingressHosts)
1923
+ REGISTRY.hosts[h] = gw;
1924
+ for (const w of LOWERED.wildcards) {
1925
+ if ("ingress" in w.target) {
1926
+ REGISTRY.wildcards.push({ suffix: wildcardSuffix(w.pattern), ip: gw });
1927
+ continue;
1928
+ }
1929
+ // Service target — only resolvable once the container has an IP.
1930
+ if (!opts.servicesUp)
1931
+ continue;
1932
+ const ip = await serviceContainerIp(w.target.service);
1933
+ if (ip) {
1934
+ REGISTRY.wildcards.push({ suffix: wildcardSuffix(w.pattern), ip });
1935
+ }
1936
+ else {
1937
+ // eslint-disable-next-line no-console
1938
+ console.warn(`[names] wildcard ${w.pattern}: service ${JSON.stringify(w.target.service)} has no IP on ${NETWORK_NAME}`);
1939
+ }
1940
+ }
1941
+ await writeRegistry();
1942
+ }
1943
+ /**
1944
+ * Register a hostname at runtime — the implementation behind `ctx.dnsName`.
1945
+ * Validates via the same `dnsName` primitive the static path uses, resolves
1946
+ * the target to an IP, and updates + persists the registry. Exact names go
1947
+ * in `hosts`; `*.suffix` wildcards in `wildcards`. The resolver re-reads on
1948
+ * the next query (it watches the file's mtime), so the name is live
1949
+ * immediately — answered for VM-host/test/browser code and for peer
1950
+ * containers (Docker forwards unknown names to the host resolver). It does
1951
+ * NOT land in any container's /etc/hosts.
1952
+ */
1953
+ async function registerDnsName(hostname, target) {
1954
+ const resv = reserveEvent();
1955
+ // Reuse the primitive purely for validation + lowercasing.
1956
+ const decl = makeDnsDecl(hostname, target);
1957
+ const ip = await resolveDnsTarget(target);
1958
+ if (isWildcard(decl.hostname)) {
1959
+ const suffix = wildcardSuffix(decl.hostname);
1960
+ REGISTRY.wildcards = REGISTRY.wildcards.filter((w) => w.suffix !== suffix);
1961
+ REGISTRY.wildcards.push({ suffix, ip });
1962
+ }
1963
+ else {
1964
+ REGISTRY.hosts[decl.hostname] = ip;
1965
+ }
1966
+ await writeRegistry();
1967
+ recordEnv({ op: "dnsName", hostname: decl.hostname, ip, durationMs: 0 }, resv);
1968
+ }
1969
+ // ────────────────────────────────────────────────────────────────────────
1970
+ // Runtime services — containers started after bootstrap (from a test, an
1971
+ // eval, project setup, or a fake handler reacting to the app under test).
1972
+ //
1973
+ // A runtime service is a *real machine on the network*: it joins
1974
+ // spectest-net with its own IP and is reached directly by name/IP, not
1975
+ // through the daemon's HTTP ingress. The same helpers bootstrap uses
1976
+ // (prepareServiceImage → runContainer → waitForReady) drive it, so it gets
1977
+ // the same image cache, CA trust, and ready-probing. Because it lives in
1978
+ // dockerd, it's captured by the per-test post-state snapshot exactly like
1979
+ // the boot services — a `dependsOn` child inherits the live container while
1980
+ // siblings (which fork from the parent's earlier snapshot) never see it.
1981
+ //
1982
+ // Tracked here only for in-VM bookkeeping (failure log capture, teardown);
1983
+ // the map forks with daemon memory, so each fork sees the services it (or
1984
+ // its ancestors) actually started.
1985
+ // ────────────────────────────────────────────────────────────────────────
1986
+ const RUNTIME_SERVICES = new Map();
1987
+ // A runtime service spec is a ServiceConfig (minus tls/dependsOn) + a name;
1988
+ // the orchestration helpers want a NamedService, which is the same shape.
1989
+ function specToNamedService(spec) {
1990
+ const { name, ...rest } = spec;
1991
+ return { name, ...rest };
1992
+ }
1993
+ /** Implementation behind `ctx.startService` / a fake's `ctx.startService`.
1994
+ * Prepares the image (pulling on first use through the host cache), runs
1995
+ * the container on spectest-net, and waits for its readyCheck. Returns the
1996
+ * container's name + IP. */
1997
+ async function startRuntimeService(spec) {
1998
+ if (!spec.name || spec.name.length === 0) {
1999
+ throw new Error("startService: `name` is required");
2000
+ }
2001
+ const t0 = Date.now();
2002
+ const resv = reserveEvent();
2003
+ const svc = specToNamedService(spec);
2004
+ const aliases = (spec.hostnames ?? []).map((h) => h.toLowerCase());
2005
+ const imageRef = svc.image.type === "registry" ? svc.image.reference : "(dockerfile)";
2006
+ try {
2007
+ const { tag } = await prepareServiceImage(svc);
2008
+ const flags = [...(await ensureVolumes(svc)), ...(await ensureFiles(svc))];
2009
+ await runContainer(svc, tag, flags, aliases);
2010
+ await waitForReady(svc);
2011
+ const ip = (await serviceContainerIp(svc.name)) ?? "";
2012
+ // `tls` is the runtime twin of a boot service's: stand up a
2013
+ // TLS-terminating reverse proxy at https://<hostname>/ → this container.
2014
+ for (const entry of svc.tls ?? []) {
2015
+ await bindRuntimeTls(entry.hostname, svc.name, entry.port);
2016
+ }
2017
+ RUNTIME_SERVICES.set(svc.name, svc);
2018
+ recordEnv({
2019
+ op: "startService",
2020
+ service: svc.name,
2021
+ image: imageRef,
2022
+ ip,
2023
+ durationMs: Date.now() - t0,
2024
+ }, resv);
2025
+ return { name: svc.name, ip };
2026
+ }
2027
+ catch (err) {
2028
+ recordEnv({
2029
+ op: "startService",
2030
+ service: svc.name,
2031
+ image: imageRef,
2032
+ durationMs: Date.now() - t0,
2033
+ error: errMessage(err),
2034
+ }, resv);
2035
+ throw err;
2036
+ }
2037
+ }
2038
+ /** Implementation behind `ctx.stopService`. Removes the container and drops
2039
+ * it from the runtime registry. No-op (rc ignored) if it's already gone. */
2040
+ async function stopRuntimeService(name) {
2041
+ const t0 = Date.now();
2042
+ const resv = reserveEvent();
2043
+ const svc = RUNTIME_SERVICES.get(name);
2044
+ await docker(["rm", "-f", name], 30_000);
2045
+ RUNTIME_SERVICES.delete(name);
2046
+ for (const entry of svc?.tls ?? [])
2047
+ await unbindRuntimeTls(entry.hostname);
2048
+ recordEnv({ op: "stopService", service: name, durationMs: Date.now() - t0 }, resv);
2049
+ }
2050
+ /** The runtime environment-control handle handed to fakes (3rd handler arg /
2051
+ * `ctx` in `helpers`). The same primitives tests get on `ctx`; module-level
2052
+ * because none of them depend on a running test. */
2053
+ const FAKE_CTX = {
2054
+ startService: startRuntimeService,
2055
+ stopService: stopRuntimeService,
2056
+ dnsName: registerDnsName,
2057
+ };
2058
+ /** Build (or fetch from cache) the helpers record for a fake — the
2059
+ * value that ends up at `ctx.fakes.<name>`. Defaults to `{}` (a fake
2060
+ * with no `helpers` exposes nothing — tests never touch private state
2061
+ * directly). Returns a tracking proxy (see `trackFakeHelpers`) so helper
2062
+ * calls land in the test timeline. */
2063
+ async function ensureFakeHelpers(name) {
2064
+ const fake = FAKES.get(name);
2065
+ if (!fake)
2066
+ throw new Error(`fake ${JSON.stringify(name)} is not loaded`);
2067
+ if (fake.trackedHelpers)
2068
+ return fake.trackedHelpers;
2069
+ fake.helpers = fake.def.helpers
2070
+ ? (await fake.def.helpers({
2071
+ name,
2072
+ state: fake.state,
2073
+ ctx: FAKE_CTX,
2074
+ }))
2075
+ : {};
2076
+ fake.trackedHelpers = trackFakeHelpers(name, fake.helpers);
2077
+ return fake.trackedHelpers;
2078
+ }
2079
+ /** Wrap a fake's helpers so each call becomes a recorded `fake` event
2080
+ * and its return value is `wrap()`ped for assertion provenance. Helpers
2081
+ * are functions that read/mutate the fake's private state via closure;
2082
+ * tests only ever see what those functions return. The proxy is built
2083
+ * once and shared across tests; it consults the recorder at call time,
2084
+ * so it's a transparent no-op when nothing is recording (eval / project
2085
+ * setup).
2086
+ *
2087
+ * Only own function properties are intercepted — inherited members
2088
+ * (`toString`, etc.), symbols, and any stray non-function property pass
2089
+ * straight through untouched. */
2090
+ function trackFakeHelpers(fakeName, helpers) {
2091
+ return new Proxy(helpers, {
2092
+ get(target, prop, receiver) {
2093
+ if (typeof prop === "symbol")
2094
+ return Reflect.get(target, prop, receiver);
2095
+ const desc = Object.getOwnPropertyDescriptor(target, prop);
2096
+ if (!desc || typeof desc.value !== "function") {
2097
+ return Reflect.get(target, prop, receiver);
2098
+ }
2099
+ const fn = desc.value;
2100
+ const member = String(prop);
2101
+ return (...args) => invokeFakeHelper(fakeName, member, fn, target, args);
2102
+ },
2103
+ });
2104
+ }
2105
+ /** Invoke a fake helper function, recording a `fake` event and wrapping
2106
+ * the return value. Handles both sync and async helpers, and records an
2107
+ * error event (then rethrows) if the helper throws. */
2108
+ function invokeFakeHelper(fakeName, member, fn, thisArg, args) {
2109
+ const t = Date.now();
2110
+ const resv = reserveEvent();
2111
+ const safeArgs = args.map((a) => safeSerialize(a));
2112
+ const recordResult = (value) => {
2113
+ const seq = recordFake({
2114
+ fake: fakeName,
2115
+ member,
2116
+ args: safeArgs,
2117
+ result: safeSerialize(value),
2118
+ durationMs: Date.now() - t,
2119
+ }, resv);
2120
+ return wrap(value, seq);
2121
+ };
2122
+ const recordError = (err) => {
2123
+ recordFake({
2124
+ fake: fakeName,
2125
+ member,
2126
+ args: safeArgs,
2127
+ durationMs: Date.now() - t,
2128
+ error: errMessage(err),
2129
+ }, resv);
2130
+ };
2131
+ let result;
2132
+ try {
2133
+ result = fn.apply(thisArg, args);
2134
+ }
2135
+ catch (err) {
2136
+ recordError(err);
2137
+ throw err;
2138
+ }
2139
+ if (result instanceof Promise) {
2140
+ return result.then(recordResult, (err) => {
2141
+ recordError(err);
2142
+ throw err;
2143
+ });
2144
+ }
2145
+ return recordResult(result);
2146
+ }
2147
+ function errMessage(err) {
2148
+ return err?.message ?? String(err);
2149
+ }
2150
+ /** Build the `fakes` map exposed on the test/eval context. Includes
2151
+ * every loaded fake; helpers are constructed lazily but we eagerly
2152
+ * materialise them here so a test can just read `ctx.fakes.x.y`. */
2153
+ async function buildFakeHandles() {
2154
+ const handles = {};
2155
+ for (const name of FAKES.keys()) {
2156
+ handles[name] = await ensureFakeHelpers(name);
2157
+ }
2158
+ return handles;
2159
+ }
2160
+ function serviceTotalMs(s) {
2161
+ return s.prepMs + (s.runMs ?? 0) + (s.readyMs ?? 0) + (s.setupMs ?? 0);
2162
+ }
2163
+ // Logged to the daemon journal (also folded into the /bootstrap response,
2164
+ // which the control plane logs). One compact line per service plus the
2165
+ // slowest BuildKit steps, so a slow cold start is profileable without
2166
+ // dumping the full build output.
2167
+ function logBootstrapTimings(t) {
2168
+ for (const s of t.services) {
2169
+ const parts = [`prep=${s.prepMs}ms(${s.kind})`];
2170
+ if (s.runMs != null)
2171
+ parts.push(`run=${s.runMs}ms`);
2172
+ if (s.readyMs != null)
2173
+ parts.push(`ready=${s.readyMs}ms`);
2174
+ if (s.setupMs)
2175
+ parts.push(`setup=${s.setupMs}ms`);
2176
+ console.log(`[bootstrap] ${s.name}: ${parts.join(" ")}`);
2177
+ if (s.buildSteps && s.buildSteps.length) {
2178
+ const top = s.buildSteps
2179
+ .map((x) => `${x.cached ? "cached" : x.secs.toFixed(1) + "s"} ${x.name}`)
2180
+ .join(" | ");
2181
+ console.log(`[bootstrap] ${s.name} build steps: ${top}`);
2182
+ }
2183
+ }
2184
+ console.log(`[bootstrap] total ${t.totalMs}ms across ${t.services.length} service(s)`);
2185
+ }
2186
+ async function bootstrap() {
2187
+ const bootStart = Date.now();
2188
+ const cfg = requireLoaded().project.environment;
2189
+ const services = namedServices(cfg);
2190
+ const timings = new Map();
2191
+ progressInit(services);
2192
+ // Build dedup is only valid within one workspace generation — a fresh
2193
+ // bootstrap may follow a workspace re-upload with the same dockerfile
2194
+ // text but different build-context content.
2195
+ BUILD_DEDUP.clear();
2196
+ // Network create is independent of the workspace-side prep, so run
2197
+ // them concurrently. .dockerignore only blocks `docker build`s — pulls
2198
+ // wouldn't need it — but the writes are sub-millisecond so we just
2199
+ // gate image prep behind both.
2200
+ await Promise.all([
2201
+ ensureNetwork(),
2202
+ (async () => {
2203
+ await fs.mkdir(WORKSPACE, { recursive: true });
2204
+ await fs.writeFile(path.join(WORKSPACE, ".dockerignore"), unionDockerignore(services));
2205
+ })(),
2206
+ ]);
2207
+ // Image prep is DECOUPLED from container start: every service's image
2208
+ // prep kicks off now, independent of `dependsOn`, and each service's
2209
+ // container start (in startServices below) gates on (its OWN image ready)
2210
+ // AND (its deps up) — there is no whole-graph barrier. So a service whose
2211
+ // image is pulled and whose deps are up starts immediately; it never sits
2212
+ // at "image ready" waiting for an unrelated slow build elsewhere.
2213
+ //
2214
+ // Prep concurrency: registry pulls always run in parallel (network-bound,
2215
+ // low VM RAM). Dockerfile builds parallelize *only* when the host
2216
+ // buildkitd is in play — there the build executes host-side under runc, so
2217
+ // N concurrent builds don't touch the VM's memory ceiling. When we fall
2218
+ // back to the in-VM builder, two or more concurrent builds routinely OOM a
2219
+ // single VM on monorepos with parallel pnpm/npm installs (each install
2220
+ // fans out to ~16 fetchers + lifecycle workers, ~70 MB/process), so we
2221
+ // serialize that case behind a FIFO chain — but only the in-VM builds
2222
+ // serialize; pulls and starts run freely alongside them. The remote-builder
2223
+ // probe is memoized, so this up-front call is free; skip it with no builds.
2224
+ const tags = new Map();
2225
+ const builds = services.filter((s) => s.image.type === "dockerfile");
2226
+ const buildsRunHostSide = builds.length > 0 && (await ensureRemoteBuilder());
2227
+ // A promise chain is a fair FIFO mutex: when builds run in-VM, each build
2228
+ // waits for the previous to settle. Pulls and host-side builds bypass it.
2229
+ let inVmBuildChain = Promise.resolve();
2230
+ const prepImage = (svc) => {
2231
+ // Bootstrap is the only dedup scope: all its builds share one
2232
+ // /workspace generation (see prepareServiceImage).
2233
+ const run = () => prepareServiceImage(svc, { dedup: true });
2234
+ if (svc.image.type === "dockerfile" && !buildsRunHostSide) {
2235
+ const next = inVmBuildChain.then(run, run);
2236
+ // Keep the chain moving even if a build throws; the chain itself never
2237
+ // rejects (the per-service prep promise below is what surfaces errors).
2238
+ inVmBuildChain = next.then(() => undefined, () => undefined);
2239
+ return next;
2240
+ }
2241
+ return run();
2242
+ };
2243
+ const prep = new Map();
2244
+ for (const svc of services) {
2245
+ const p = (async () => {
2246
+ const t0 = Date.now();
2247
+ const { tag, buildSteps } = await prepImage(svc);
2248
+ progressService(svc.name, { status: "prepared", detail: undefined });
2249
+ tags.set(svc.name, tag);
2250
+ timings.set(svc.name, {
2251
+ name: svc.name,
2252
+ kind: svc.image.type === "registry" ? "pull" : "build",
2253
+ prepMs: Date.now() - t0,
2254
+ buildSteps,
2255
+ });
2256
+ })();
2257
+ // A dependent whose dep fails aborts before it awaits this prep, which
2258
+ // would leave the prep promise unobserved. Attach a no-op handler so a
2259
+ // late rejection can't crash the daemon; startOne still re-throws it for
2260
+ // services that do reach their await.
2261
+ p.catch(() => undefined);
2262
+ prep.set(svc.name, p);
2263
+ }
2264
+ // Ingress (fakes + service-tls proxies) comes up BEFORE services so
2265
+ // that any service that calls a fake URL during its own startup
2266
+ // probe finds it answering. Service-tls proxies will return 502
2267
+ // until their upstream containers start, but no one is hitting
2268
+ // https://<svc>.test/ during bootstrap so that's harmless. The
2269
+ // bridge gateway IP is set on `network create` — independent of
2270
+ // any container being up — so we don't need services to determine
2271
+ // the listener address.
2272
+ await startIngress();
2273
+ progressPhase("Starting services");
2274
+ // Container start + ready probe driven by the dependsOn DAG: each
2275
+ // service starts the moment its own dependencies finish run→probe→setup,
2276
+ // instead of waiting for a whole topological level to clear. We chain
2277
+ // run→probe→setup per service so a dependent sees the post-setup state
2278
+ // of its deps (a database with its schema applied, a k3s cluster with
2279
+ // its ingress controller already running) — but an unrelated slow probe
2280
+ // no longer holds back a branch that's ready to go.
2281
+ await startServices(services, async (svc) => {
2282
+ // Gate on our OWN image being ready. startServices already gated on our
2283
+ // deps; this adds the image edge. The two compose: we run the moment
2284
+ // both are satisfied, with no whole-graph barrier between them.
2285
+ await prep.get(svc.name);
2286
+ const flags = [...(await ensureVolumes(svc)), ...(await ensureFiles(svc))];
2287
+ const tag = tags.get(svc.name);
2288
+ if (!tag)
2289
+ throw new Error(`internal: no image tag for ${svc.name}`);
2290
+ const tRun = Date.now();
2291
+ progressService(svc.name, { status: "starting", detail: undefined });
2292
+ await runContainer(svc, tag, flags);
2293
+ const tReady = Date.now();
2294
+ progressService(svc.name, { status: "probing", detail: "ready check" });
2295
+ await waitForReady(svc);
2296
+ const tSetup = Date.now();
2297
+ if (svc.setup) {
2298
+ progressService(svc.name, { status: "probing", detail: "running setup" });
2299
+ const helpers = await ensureHelpers(svc.name, svc);
2300
+ await svc.setup({ name: svc.name, helpers, ...componentContext() });
2301
+ }
2302
+ progressService(svc.name, { status: "ready", detail: undefined });
2303
+ const ti = timings.get(svc.name);
2304
+ if (ti) {
2305
+ ti.runMs = tReady - tRun;
2306
+ ti.readyMs = tSetup - tReady;
2307
+ ti.setupMs = svc.setup ? Date.now() - tSetup : 0;
2308
+ }
2309
+ });
2310
+ // Containers now have IPs — re-seed so service-targeted wildcards (e.g.
2311
+ // k3s ingressDomains → the cluster container) resolve. Captured into the
2312
+ // warm template, so warm starts inherit the resolved entries.
2313
+ await seedNamesRegistry({ servicesUp: true });
2314
+ // Browser pre-warm DISABLED (2026-06-08). We used to pre-open one view
2315
+ // into the pool (browser.ts VIEW_POOL) here so every fork inherited a
2316
+ // live renderer and the first ctx.browser() skipped the ~1.2-1.5s spawn.
2317
+ // But a renderer spawned BEFORE the snapshot and restored in a fork holds
2318
+ // stale DNS state: its first navigate to an ingress host fails
2319
+ // `net::ERR_NAME_NOT_RESOLVED` even though getaddrinfo/fetch resolve fine
2320
+ // (the --disable-features=AsyncDns flag doesn't save the pooled view). A
2321
+ // view created fresh AFTER the fork (openBrowser → createView, since the
2322
+ // pool is now empty) spawns a post-restore renderer with correct DNS. The
2323
+ // tradeoff is the per-test spawn cost is back on the browser path; we
2324
+ // accept it to keep the suite's browser-rooted DAGs working. See
2325
+ // browser.ts:213 (the long-standing intermittent NAME_NOT_RESOLVED) and
2326
+ // the clocksource-regression notes. Re-enabling requires fixing the
2327
+ // restored-renderer DNS state, not just re-adding the prewarm call.
2328
+ // NOTE: persistent sessions (ctx.browser/ctx.mobile keep one live view
2329
+ // across tests, so restored forks navigate on a pre-snapshot renderer
2330
+ // routinely) hit the same bug head-on; browser.ts handles it there by
2331
+ // rebuilding the view in the same Chrome and retrying the navigation
2332
+ // (`rebuildView`) — profile state survives, so auth carries over. That
2333
+ // recovery is scoped to inherited-navigation failures and does NOT make
2334
+ // the about:blank prewarm pool safe to re-enable.
2335
+ const result = {
2336
+ totalMs: Date.now() - bootStart,
2337
+ services: [...timings.values()].sort((a, b) => serviceTotalMs(b) - serviceTotalMs(a)),
2338
+ };
2339
+ progressDone();
2340
+ logBootstrapTimings(result);
2341
+ return result;
2342
+ }
2343
+ /**
2344
+ * Run the loaded project's `setup` hook, if any. Called by the control
2345
+ * plane once between /bootstrap and the warm-template snapshot, so the
2346
+ * effects (seeded DB rows, initial pods, fixture files) are captured
2347
+ * exactly once and inherited by every later snapshot/fork.
2348
+ *
2349
+ * Unlike test runs, this is NOT instrumented — no recorder, no event
2350
+ * timeline, no timeout from the test runner. Setup failures abort the
2351
+ * env bring-up; the control plane surfaces them as a start failure.
2352
+ */
2353
+ async function runProjectSetup() {
2354
+ const proj = requireLoaded().project;
2355
+ if (!proj.setup)
2356
+ return { ran: false, durationMs: 0 };
2357
+ const start = Date.now();
2358
+ // Build the same `svc` handles tests see, so setup and tests share
2359
+ // helper instances (e.g. a Bun.SQL pool created here is reused later).
2360
+ const svc = (await buildServiceHandles(proj.environment));
2361
+ const fakes = await buildFakeHandles();
2362
+ // Install the fetch wrapper for the duration of setup so `ctx.fetch` (and any
2363
+ // client routed through `globalThis.fetch`) returns a wrapped Response, same
2364
+ // as in a test. No recorder is active here, so it wraps without provenance —
2365
+ // but the wrapped type stays honest at runtime (`.unwrap()` works).
2366
+ const restoreFetch = installFetchWrapper();
2367
+ const ctx = {
2368
+ fetch: globalThis.fetch,
2369
+ exec: execInServiceWrapped,
2370
+ svc,
2371
+ fakes,
2372
+ dnsName: registerDnsName,
2373
+ startService: startRuntimeService,
2374
+ stopService: stopRuntimeService,
2375
+ };
2376
+ try {
2377
+ await proj.setup(ctx);
2378
+ }
2379
+ finally {
2380
+ restoreFetch();
2381
+ }
2382
+ return { ran: true, durationMs: Date.now() - start };
2383
+ }
2384
+ // ────────────────────────────────────────────────────────────────────────
2385
+ // Replay bundles (rrweb sessions → gzipped side-channel, pulled in chunks)
2386
+ // ────────────────────────────────────────────────────────────────────────
2387
+ //
2388
+ // `RunResult.browserSessions` never rides the `/run` reply: the vm-agent
2389
+ // caps proxied daemon responses at 16 MB and a browser-heavy case's
2390
+ // recording is tens of MB of JSON (see replay-bundle.ts for the whole
2391
+ // story). Instead the /run handler encodes the sessions into a gzipped,
2392
+ // asset-deduplicated bundle, parks it here keyed by case id, and replies
2393
+ // with a tiny `replay: { bytes }` ref; the control plane then pulls the
2394
+ // bundle via `POST /replay-chunk` in base64 chunks sized under the cap,
2395
+ // before it tears the fork down.
2396
+ //
2397
+ // The map is module memory, so it FORKS with the snapshot (the post-test
2398
+ // snapshot is captured before the control plane fetches). The size cap
2399
+ // keeps a long handoff chain from accreting every ancestor's (gzipped)
2400
+ // bundle in guest RAM — entries older than the last few are always
2401
+ // already-fetched leftovers frozen into some snapshot.
2402
+ const REPLAY_BUNDLES = new Map();
2403
+ const REPLAY_BUNDLES_MAX = 8;
2404
+ function stashReplayBundle(caseId, gz) {
2405
+ // Re-insert to refresh recency (Map iterates in insertion order).
2406
+ REPLAY_BUNDLES.delete(caseId);
2407
+ REPLAY_BUNDLES.set(caseId, gz);
2408
+ while (REPLAY_BUNDLES.size > REPLAY_BUNDLES_MAX) {
2409
+ const oldest = REPLAY_BUNDLES.keys().next().value;
2410
+ if (oldest === undefined)
2411
+ break;
2412
+ REPLAY_BUNDLES.delete(oldest);
2413
+ }
2414
+ }
2415
+ // ────────────────────────────────────────────────────────────────────────
2416
+ // Cumulative service logs (per-case deltas → S3, reconstructed on the web)
2417
+ // ────────────────────────────────────────────────────────────────────────
2418
+ //
2419
+ // A child test runs in a fork restored from its parent's memory+filesystem
2420
+ // snapshot, so `docker logs <svc>` in the child already contains the
2421
+ // parent's entire history plus the child's own output — logs are
2422
+ // inherently cumulative along each branch. Rather than store the (growing,
2423
+ // redundant) full log at every case, each case stores only its DELTA (the
2424
+ // lines it added), and the dashboard reconstructs a branch's full log by
2425
+ // concatenating deltas along the ancestor chain.
2426
+ //
2427
+ // The load-bearing trick: `LOG_MARKERS` lives in module memory, so it
2428
+ // FORKS with the snapshot (same mechanism as `TEST_DATA` / `RUNTIME_SERVICES`).
2429
+ // `runOne` advances the markers BEFORE `/run` returns, and the control
2430
+ // plane snapshots the fork AFTER `/run` returns — so a child (forked or
2431
+ // handed off) restores the parent's final markers and its delta tiles on
2432
+ // with no gap or cross-branch duplication.
2433
+ /**
2434
+ * Per-(service, stream) count of newline-terminated log lines already
2435
+ * captured by an ancestor case. Key is `"<service><stream>"`. The
2436
+ * next case on this branch captures only `lines[marker..]`. Module-scope
2437
+ * so it travels with the snapshot into every fork.
2438
+ */
2439
+ const LOG_MARKERS = new Map();
2440
+ /** Per-(service, stream) delta byte cap. Over this we keep head+tail and
2441
+ * elide the middle — the head preserves the continuation from the parent,
2442
+ * the tail preserves the newest output — while still advancing the marker
2443
+ * to the true line count so the chain stays aligned. */
2444
+ const LOG_DELTA_MAX_BYTES = 2 * 1024 * 1024;
2445
+ /**
2446
+ * Inspect a container's run state for {@link ServiceLogDelta}: `{}` while
2447
+ * running (or when inspect fails — a removed container has no state left to
2448
+ * report), `{ stopped, exitCode }` once it has exited/died.
2449
+ */
2450
+ async function containerStopState(name) {
2451
+ const r = await docker(["inspect", "-f", "{{.State.Status}} {{.State.ExitCode}}", name], 15_000);
2452
+ if (r.code !== 0)
2453
+ return {};
2454
+ const [status, codeStr] = r.stdout.trim().split(/\s+/);
2455
+ if (status !== "exited" && status !== "dead")
2456
+ return {};
2457
+ const exitCode = Number.parseInt(codeStr ?? "", 10);
2458
+ return Number.isFinite(exitCode) ? { stopped: true, exitCode } : { stopped: true };
2459
+ }
2460
+ /** Keep the head and tail of `s`, eliding the middle when it exceeds
2461
+ * `max` (string length, a byte proxy as elsewhere here). Head+tail so an
2462
+ * over-long delta keeps both the parent-continuation and the newest
2463
+ * output. */
2464
+ function capMiddle(s, max) {
2465
+ if (s.length <= max)
2466
+ return { value: s, truncated: false };
2467
+ const half = Math.floor(max / 2);
2468
+ const elided = s.length - 2 * half;
2469
+ return {
2470
+ value: `${s.slice(0, half)}\n… [${elided} bytes elided] …\n${s.slice(s.length - half)}`,
2471
+ truncated: true,
2472
+ };
2473
+ }
2474
+ /**
2475
+ * Compute one stream's delta beyond `marker` complete lines.
2476
+ * - Counts only newline-terminated lines; a trailing partial line (no
2477
+ * `\n` yet) is held back from both the delta and the count, so a line
2478
+ * completed by a later capture isn't split across the fork boundary.
2479
+ * - Reset guard: if the stream shrank below `marker` (container recreated
2480
+ * or rotated) the whole current log is re-emitted and `reset` is set.
2481
+ */
2482
+ function streamDelta(full, marker) {
2483
+ const lastNl = full.lastIndexOf("\n");
2484
+ const complete = lastNl < 0 ? "" : full.slice(0, lastNl + 1);
2485
+ let total = 0;
2486
+ for (let i = 0; i < complete.length; i++) {
2487
+ if (complete.charCodeAt(i) === 10)
2488
+ total++;
2489
+ }
2490
+ let reset = false;
2491
+ let startLine = marker;
2492
+ if (total < marker) {
2493
+ reset = true;
2494
+ startLine = 0;
2495
+ }
2496
+ let delta;
2497
+ if (startLine <= 0) {
2498
+ delta = complete;
2499
+ }
2500
+ else if (startLine >= total) {
2501
+ delta = "";
2502
+ }
2503
+ else {
2504
+ // Byte offset just past the `startLine`-th newline.
2505
+ let seen = 0;
2506
+ let off = 0;
2507
+ for (let i = 0; i < complete.length; i++) {
2508
+ if (complete.charCodeAt(i) === 10 && ++seen === startLine) {
2509
+ off = i + 1;
2510
+ break;
2511
+ }
2512
+ }
2513
+ delta = complete.slice(off);
2514
+ }
2515
+ const capped = capMiddle(delta, LOG_DELTA_MAX_BYTES);
2516
+ return { delta: capped.value, total, reset, truncated: capped.truncated };
2517
+ }
2518
+ /**
2519
+ * Capture the per-service log delta for the current case and advance the
2520
+ * markers. Runs on EVERY case (pass or fail). Enumerates boot services
2521
+ * (`namedServices`) plus any runtime services this fork started
2522
+ * (`RUNTIME_SERVICES`), deduped by name. A `docker logs` failure surfaces
2523
+ * as the service's `stderr` WITHOUT advancing the markers — a transient
2524
+ * failure must never desync the chain.
2525
+ */
2526
+ async function captureServiceLogDeltas() {
2527
+ const l = loaded;
2528
+ if (!l)
2529
+ return [];
2530
+ const byName = new Map();
2531
+ for (const s of namedServices(l.project.environment))
2532
+ byName.set(s.name, s);
2533
+ for (const [name, s] of RUNTIME_SERVICES)
2534
+ byName.set(name, s);
2535
+ const services = [...byName.values()];
2536
+ return Promise.all(services.map(async (svc) => {
2537
+ const [r, stop] = await Promise.all([
2538
+ docker(["logs", "--timestamps", svc.name], 30_000),
2539
+ containerStopState(svc.name),
2540
+ ]);
2541
+ if (r.code !== 0) {
2542
+ // Container gone/renamed — surface the CLI error, leave markers put.
2543
+ const err = capMiddle(r.stderr || r.stdout, LOG_DELTA_MAX_BYTES);
2544
+ return {
2545
+ service: svc.name,
2546
+ stdout: "",
2547
+ stdoutTruncated: false,
2548
+ stdoutReset: false,
2549
+ stderr: err.value,
2550
+ stderrTruncated: err.truncated,
2551
+ stderrReset: false,
2552
+ ...stop,
2553
+ };
2554
+ }
2555
+ const outKey = `${svc.name}stdout`;
2556
+ const errKey = `${svc.name}stderr`;
2557
+ const out = streamDelta(r.stdout, LOG_MARKERS.get(outKey)?.lines ?? 0);
2558
+ const err = streamDelta(r.stderr, LOG_MARKERS.get(errKey)?.lines ?? 0);
2559
+ LOG_MARKERS.set(outKey, { lines: out.total });
2560
+ LOG_MARKERS.set(errKey, { lines: err.total });
2561
+ return {
2562
+ service: svc.name,
2563
+ stdout: out.delta,
2564
+ stdoutTruncated: out.truncated,
2565
+ stdoutReset: out.reset,
2566
+ stderr: err.delta,
2567
+ stderrTruncated: err.truncated,
2568
+ stderrReset: err.reset,
2569
+ ...stop,
2570
+ };
2571
+ }));
2572
+ }
2573
+ /**
2574
+ * Mint a session id. `idScope` (the running test's case id; `"eval"`
2575
+ * for eval-context sessions) is baked in because `randomUUID()` alone
2576
+ * is NOT unique across test forks: sibling cases resume from the same
2577
+ * snapshot, so the daemon process — and the guest kernel CSPRNG it
2578
+ * draws from — restores identical RNG state in every clone, and the
2579
+ * first UUID minted after the fork collides across siblings (observed
2580
+ * in practice, not hypothetical). Persistence keys sessions by
2581
+ * (run, case, session) so the collision never lost data, but anything
2582
+ * that ever aggregates sessions across cases would conflate them.
2583
+ * Sibling forks run different cases by construction, so the case id is
2584
+ * exactly the entropy the clones are missing.
2585
+ */
2586
+ function newSessionId(idScope) {
2587
+ return idScope ? `${idScope}:${randomUUID()}` : randomUUID();
2588
+ }
2589
+ /**
2590
+ * Cumulative raw-byte cap on one eval's artifacts. The bytes ride the
2591
+ * `/eval` JSON reply base64'd (~1.33x), and the vm-agent hard-errors on
2592
+ * proxied daemon responses over 16 MB (the same cap that pushed /run's
2593
+ * replay bundles out-of-band — see REPLAY_BUNDLES). 8 MiB raw ≈ 10.7 MB
2594
+ * encoded leaves headroom for the reply's sessions/log; screenshots are
2595
+ * typically well under 2 MiB each. If artifacts ever need to grow past
2596
+ * this, move them to the parked-chunk side channel instead of raising it.
2597
+ */
2598
+ const MAX_EVAL_ARTIFACT_BYTES = 8 * 1024 * 1024;
2599
+ /**
2600
+ * Per-eval artifact sink. `register` throws (failing the screenshot() call,
2601
+ * never the eval) once the byte budget is exhausted.
2602
+ */
2603
+ function newArtifactCollector() {
2604
+ const artifacts = [];
2605
+ let total = 0;
2606
+ return {
2607
+ artifacts,
2608
+ register(artifact) {
2609
+ total += artifact.sizeBytes;
2610
+ if (total > MAX_EVAL_ARTIFACT_BYTES) {
2611
+ throw new Error(`artifact byte budget for this eval exceeded (${MAX_EVAL_ARTIFACT_BYTES / (1024 * 1024)} MiB) — ` +
2612
+ "capture fewer screenshots per eval");
2613
+ }
2614
+ artifacts.push(artifact);
2615
+ },
2616
+ };
2617
+ }
2618
+ /**
2619
+ * Build the recorder sink + bookkeeping for a single Browser session.
2620
+ * The returned `recorder` is what `openBrowser` writes into; the
2621
+ * returned `record` is the in-flight session object the daemon owns.
2622
+ * `artifacts` (eval-only) wires `screenshot()`'s artifact registration —
2623
+ * test-run sessions don't pass it, which is exactly what makes
2624
+ * `screenshot()` throw outside eval.
2625
+ */
2626
+ function newBrowserSession(testStart, idScope, frame = "browser", artifacts) {
2627
+ const record = {
2628
+ sessionId: newSessionId(idScope),
2629
+ openedAtMs: Date.now() - testStart,
2630
+ frame,
2631
+ steps: [],
2632
+ };
2633
+ let closed = false;
2634
+ return {
2635
+ record,
2636
+ recorder: {
2637
+ sessionId: record.sessionId,
2638
+ recordStep(step) {
2639
+ if (closed)
2640
+ return;
2641
+ record.steps.push(step);
2642
+ },
2643
+ noteNavigation(url) {
2644
+ if (closed)
2645
+ return;
2646
+ if (record.initialUrl === undefined)
2647
+ record.initialUrl = url;
2648
+ },
2649
+ ...(artifacts
2650
+ ? { registerArtifact: (a) => artifacts.register(a) }
2651
+ : {}),
2652
+ },
2653
+ markClosed() {
2654
+ if (closed)
2655
+ return;
2656
+ closed = true;
2657
+ record.closedAtMs = Date.now() - testStart;
2658
+ },
2659
+ };
2660
+ }
2661
+ /** New terminal session bookkeeping for a single `ctx.terminal(...)` call. */
2662
+ function newTerminalSession(testStart, service, command, cols, rows, idScope) {
2663
+ const record = {
2664
+ sessionId: newSessionId(idScope),
2665
+ openedAtMs: Date.now() - testStart,
2666
+ service,
2667
+ command,
2668
+ cols,
2669
+ rows,
2670
+ frames: [],
2671
+ };
2672
+ let closed = false;
2673
+ return {
2674
+ record,
2675
+ pushFrame(tSec, data) {
2676
+ if (closed)
2677
+ return;
2678
+ record.frames.push([tSec, "o", data]);
2679
+ },
2680
+ markClosed() {
2681
+ if (closed)
2682
+ return;
2683
+ closed = true;
2684
+ record.closedAtMs = Date.now() - testStart;
2685
+ },
2686
+ };
2687
+ }
2688
+ /** Grid a recorded `ctx.exec` asciicast claims. There's no PTY behind
2689
+ * an exec so no real size exists — 80×24 matches the `ctx.terminal`
2690
+ * default, and the player hard-wraps longer lines the way an actual
2691
+ * 80-col terminal would. */
2692
+ const EXEC_CAST_COLS = 80;
2693
+ const EXEC_CAST_ROWS = 24;
2694
+ /** Cumulative cap on asciicast frame bytes per recorded exec. The exec
2695
+ * *event* caps its stdout/stderr separately (256 KiB each); this bounds
2696
+ * the recording, which would otherwise duplicate a huge output in the
2697
+ * run payload and the DB. On overflow the cast gets one trailing
2698
+ * notice frame and stops growing; the ExecResult is unaffected. */
2699
+ const EXEC_FRAME_CAP_BYTES = 1024 * 1024;
2700
+ /**
2701
+ * Build the bookkeeping for a fresh `openTerminal` session and return
2702
+ * the open Terminal handle alongside a frame sink the factory drains
2703
+ * into. The daemon owns the `TerminalSessionRecord`; the factory just
2704
+ * pushes frames and tells us when the session ends.
2705
+ *
2706
+ * Used by both:
2707
+ * - the long-lived `ctx.openTerminal(...)` API, where the test owns
2708
+ * the handle and decides when to close;
2709
+ * - the one-shot `ctx.terminal(...)` wrapper below, which opens a
2710
+ * terminal with `opts.command`, waits for the embedded program to
2711
+ * exit, then closes — same code path, just an immediate await.
2712
+ *
2713
+ * Heads-up for test authors: TTY-detecting CLIs may invoke a pager
2714
+ * (psql → less, git → less, etc.) and block waiting for input now that
2715
+ * stdin *is* a TTY. Disable paging in the command itself (e.g.
2716
+ * `psql -P pager=off`) or pass `PAGER=cat` / `PSQL_PAGER=` via
2717
+ * `opts.env`.
2718
+ */
2719
+ async function openInstrumentedTerminal(service, opts, testStart, sessions, recordEvents, idScope) {
2720
+ const cols = opts?.cols ?? 80;
2721
+ const rows = opts?.rows ?? 24;
2722
+ const session = newTerminalSession(testStart, service, opts?.command ?? "(interactive)", cols, rows, idScope);
2723
+ sessions.push(session.record);
2724
+ const sink = {
2725
+ pushFrame: (t, data) => session.pushFrame(t, data),
2726
+ markClosed: () => session.markClosed(),
2727
+ };
2728
+ return await openTerminal({
2729
+ service,
2730
+ opts,
2731
+ sink,
2732
+ sessionId: session.record.sessionId,
2733
+ recordEvents,
2734
+ });
2735
+ }
2736
+ // Return value of each test that has completed in this daemon's lifetime.
2737
+ // Lives in daemon memory and is captured by every post-test snapshot, so
2738
+ // when a child case forks from its parent's snapshot it sees the same Map
2739
+ // already populated. Carries arbitrary JS values — no JSON round-trip.
2740
+ const TEST_DATA = new Map();
2741
+ // ────────────────────────────────────────────────────────────────────────
2742
+ // Component context — the exec / project-file surface handed to service
2743
+ // `setup` hooks and `helpers` factories (`ComponentContext` in index.ts),
2744
+ // so components don't hand-roll child_process docker execs or hard-code
2745
+ // control-plane paths like /workspace.
2746
+ // ────────────────────────────────────────────────────────────────────────
2747
+ const COMPONENT_EXEC_DEFAULT_TIMEOUT_MS = 120_000;
2748
+ /** Raw `docker exec` with optional piped stdin. Array command = exact
2749
+ * argv (no shell); string = `sh -lc`. Non-zero exit is reported via
2750
+ * `exitCode`, never thrown. Unlike `execInService` this records nothing —
2751
+ * setup/helpers-factory time has no test timeline. */
2752
+ function componentExec(service, command, opts) {
2753
+ const argv = ["exec", "-i"];
2754
+ if (opts?.cwd)
2755
+ argv.push("-w", opts.cwd);
2756
+ argv.push(service);
2757
+ if (typeof command === "string")
2758
+ argv.push("sh", "-lc", command);
2759
+ else
2760
+ argv.push(...command);
2761
+ const timeoutMs = opts?.timeoutMs ?? COMPONENT_EXEC_DEFAULT_TIMEOUT_MS;
2762
+ return new Promise((resolve, reject) => {
2763
+ const child = spawn("docker", argv, {
2764
+ stdio: [opts?.stdin !== undefined ? "pipe" : "ignore", "pipe", "pipe"],
2765
+ });
2766
+ const out = [];
2767
+ const err = [];
2768
+ child.stdout.on("data", (c) => out.push(c));
2769
+ child.stderr.on("data", (c) => err.push(c));
2770
+ const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
2771
+ child.on("error", (e) => {
2772
+ clearTimeout(timer);
2773
+ reject(e);
2774
+ });
2775
+ child.on("close", (code) => {
2776
+ clearTimeout(timer);
2777
+ resolve({
2778
+ stdout: Buffer.concat(out).toString("utf8"),
2779
+ stderr: Buffer.concat(err).toString("utf8"),
2780
+ exitCode: code ?? -1,
2781
+ });
2782
+ });
2783
+ if (opts?.stdin !== undefined)
2784
+ child.stdin.end(opts.stdin);
2785
+ });
2786
+ }
2787
+ function componentContext() {
2788
+ return {
2789
+ projectRoot: WORKSPACE,
2790
+ readProjectFile: (p) => fs.readFile(path.isAbsolute(p) ? p : path.join(WORKSPACE, p), "utf8"),
2791
+ exec: componentExec,
2792
+ };
2793
+ }
2794
+ // Cached helper namespaces produced by `ServiceDefinition.helpers`
2795
+ // factories. Built lazily on first access and reused for the daemon's
2796
+ // lifetime — Bun.SQL pools and similar resources are happy to live a
2797
+ // long time, and the underlying TCP connections survive snapshot/fork
2798
+ // along with the rest of daemon memory. Cleared on /load and /reload
2799
+ // (project change invalidates any cached state).
2800
+ const HELPERS_CACHE = new Map();
2801
+ /**
2802
+ * Build (or fetch from cache) the helpers record for a single service.
2803
+ * Returns an empty object if the service doesn't ship a `helpers`
2804
+ * factory — symmetric with what gets passed to `setup`.
2805
+ */
2806
+ async function ensureHelpers(name, def) {
2807
+ if (!def.helpers)
2808
+ return {};
2809
+ if (!HELPERS_CACHE.has(name)) {
2810
+ HELPERS_CACHE.set(name, await def.helpers({ name, ...componentContext() }));
2811
+ }
2812
+ return HELPERS_CACHE.get(name);
2813
+ }
2814
+ /**
2815
+ * Build the `svc` map for one test/eval. The value at `svc[name]` is
2816
+ * exactly the record the service's `helpers` factory returned (e.g.
2817
+ * `{ client: SqlClient }` for `postgres(...)`). Services without a
2818
+ * `helpers` factory don't appear in the map at all.
2819
+ */
2820
+ async function buildServiceHandles(cfg) {
2821
+ const handles = {};
2822
+ for (const [name, rawDef] of Object.entries(cfg.services)) {
2823
+ // The wire type drops the `helpers` function (JSON.stringify ignores
2824
+ // functions), but in the daemon we hold the in-memory definition
2825
+ // from the user's module — `helpers` is still there when present.
2826
+ const def = rawDef;
2827
+ if (!def.helpers)
2828
+ continue;
2829
+ handles[name] = await ensureHelpers(name, def);
2830
+ }
2831
+ return handles;
2832
+ }
2833
+ // ────────────────────────────────────────────────────────────────────────
2834
+ // Instrumentation helpers
2835
+ // ────────────────────────────────────────────────────────────────────────
2836
+ function describeFetchInput(input) {
2837
+ if (typeof input === "string")
2838
+ return { url: input };
2839
+ if (input instanceof URL)
2840
+ return { url: input.toString() };
2841
+ // Request instance
2842
+ const req = input;
2843
+ return { url: req.url, methodFromInput: req.method };
2844
+ }
2845
+ function describeRequestBody(input, init) {
2846
+ // For Request objects, body has already been consumed into the request;
2847
+ // we can't read it back without cloning, which costs. Skip unless init.body
2848
+ // is provided directly.
2849
+ const body = init?.body;
2850
+ if (body === undefined || body === null) {
2851
+ if (input instanceof Request && input.bodyUsed === false) {
2852
+ // Don't drain the request's body here — leaving it for the actual
2853
+ // fetch. Return a marker.
2854
+ return { body: "[Request body not captured]", truncated: false };
2855
+ }
2856
+ return {};
2857
+ }
2858
+ if (typeof body === "string") {
2859
+ const t = truncateUtf8(body);
2860
+ return { body: t.value, truncated: t.truncated };
2861
+ }
2862
+ if (body instanceof URLSearchParams) {
2863
+ const t = truncateUtf8(body.toString());
2864
+ return { body: t.value, truncated: t.truncated };
2865
+ }
2866
+ return { body: `[non-text body: ${body.constructor?.name ?? typeof body}]` };
2867
+ }
2868
+ /**
2869
+ * Install a fetch wrapper on `globalThis` that emits HTTP events into the
2870
+ * active recorder. Returns a restore function. Calls outside of a running
2871
+ * test still hit the original fetch (the recorder is null then; the
2872
+ * wrapper just adds a tiny amount of overhead — but we restore after each
2873
+ * test anyway, so this only matters mid-test).
2874
+ */
2875
+ function installFetchWrapper() {
2876
+ const original = globalThis.fetch;
2877
+ const wrappedFn = async (input, init) => {
2878
+ const start = Date.now();
2879
+ const resv = reserveEvent();
2880
+ const { url, methodFromInput } = describeFetchInput(input);
2881
+ const method = (init?.method ?? methodFromInput ?? "GET").toUpperCase();
2882
+ const reqBody = describeRequestBody(input, init);
2883
+ try {
2884
+ const res = await original(input, init);
2885
+ let responseBody;
2886
+ let responseBodyTruncated;
2887
+ try {
2888
+ const cloned = res.clone();
2889
+ const text = await cloned.text();
2890
+ const t = truncateUtf8(text);
2891
+ responseBody = t.value;
2892
+ responseBodyTruncated = t.truncated;
2893
+ }
2894
+ catch {
2895
+ // Binary or unreadable body — leave undefined.
2896
+ }
2897
+ const seq = recordHttp({
2898
+ method,
2899
+ url,
2900
+ requestBody: reqBody.body,
2901
+ requestBodyTruncated: reqBody.truncated,
2902
+ status: res.status,
2903
+ responseBody,
2904
+ responseBodyTruncated,
2905
+ durationMs: Date.now() - start,
2906
+ }, resv);
2907
+ return wrapResponse(res, seq);
2908
+ }
2909
+ catch (err) {
2910
+ const e = err;
2911
+ recordHttp({
2912
+ method,
2913
+ url,
2914
+ requestBody: reqBody.body,
2915
+ requestBodyTruncated: reqBody.truncated,
2916
+ durationMs: Date.now() - start,
2917
+ error: e?.message ?? String(err),
2918
+ }, resv);
2919
+ throw err;
2920
+ }
2921
+ };
2922
+ // Preserve any provider-specific statics on `fetch` (e.g. Bun's
2923
+ // `fetch.preconnect`) so consumers that touch them keep working.
2924
+ const wrapped = wrappedFn;
2925
+ for (const key of Object.keys(original)) {
2926
+ wrapped[key] = original[key];
2927
+ }
2928
+ globalThis.fetch = wrapped;
2929
+ return () => {
2930
+ globalThis.fetch = original;
2931
+ };
2932
+ }
2933
+ /** Build the `docker exec` argv for a service command. An optional `cwd`
2934
+ * becomes `-w <cwd>` (the working-directory option of `ctx.exec`), so the
2935
+ * command runs from that directory without it being baked into the command
2936
+ * string. Used by both the buffered and streaming variants so they stay in
2937
+ * lockstep. */
2938
+ function dockerExecArgs(service, command, cwd) {
2939
+ const args = ["exec"];
2940
+ if (cwd)
2941
+ args.push("-w", cwd);
2942
+ args.push(service, "sh", "-lc", command);
2943
+ return args;
2944
+ }
2945
+ function execInService(service, command, cwd) {
2946
+ return new Promise((resolve) => {
2947
+ execFile("docker", dockerExecArgs(service, command, cwd), { maxBuffer: 16 * 1024 * 1024 }, (err, stdout, stderr) => {
2948
+ const exitCode = err && typeof err.code === "number"
2949
+ ? Number(err.code)
2950
+ : err
2951
+ ? 1
2952
+ : 0;
2953
+ resolve({
2954
+ stdout: stdout.toString(),
2955
+ stderr: stderr.toString(),
2956
+ exitCode,
2957
+ });
2958
+ });
2959
+ });
2960
+ }
2961
+ /** `execInService` for the no-recorder contexts (`setup`/`eval`): wraps the
2962
+ * result so `.unwrap()` is available and the ctx's wrapped `exec` type is
2963
+ * honest at runtime, but with no provenance (there's no event to link to).
2964
+ * The recorded `ctx.exec` used during tests is `recordedExec` below. */
2965
+ async function execInServiceWrapped(service, command, opts) {
2966
+ const res = await execInService(service, command, opts?.cwd);
2967
+ return wrap(res, undefined);
2968
+ }
2969
+ /** Per-stream cap on the ExecResult strings the streaming variant
2970
+ * accumulates — parity with the buffered `execInService`'s `maxBuffer`.
2971
+ * Past the cap we keep draining (so the child never blocks on a full
2972
+ * pipe) but stop appending; unlike `execFile` we don't kill the
2973
+ * process, which only makes over-cap runs *more* survivable. */
2974
+ const EXEC_RESULT_CAP_BYTES = 16 * 1024 * 1024;
2975
+ /**
2976
+ * Streaming variant of `execInService` for the recorded `ctx.exec`:
2977
+ * the same `docker exec <svc> sh -lc <cmd>` invocation with the same
2978
+ * result shape, but stdout/stderr are drained incrementally so the
2979
+ * caller can timestamp each chunk into an asciicast frame as it
2980
+ * arrives. No PTY is involved — the program still sees plain pipes
2981
+ * (`isatty` false), so the streams stay byte-identical to what `exec`
2982
+ * has always returned; the recording adds arrival *timing* only.
2983
+ * `onChunk` fires in arrival order across both streams — the closest
2984
+ * analogue of what a terminal would have shown — while the returned
2985
+ * `ExecResult` keeps them separate as before.
2986
+ */
2987
+ function execInServiceStreaming(service, command, onChunk, cwd) {
2988
+ return new Promise((resolve) => {
2989
+ const child = spawn("docker", dockerExecArgs(service, command, cwd), {
2990
+ stdio: ["ignore", "pipe", "pipe"],
2991
+ });
2992
+ const acc = { stdout: "", stderr: "" };
2993
+ const decoders = {
2994
+ stdout: new TextDecoder("utf-8", { fatal: false }),
2995
+ stderr: new TextDecoder("utf-8", { fatal: false }),
2996
+ };
2997
+ const drain = (which, chunk) => {
2998
+ const data = decoders[which].decode(chunk, { stream: true });
2999
+ if (data.length === 0)
3000
+ return;
3001
+ if (acc[which].length < EXEC_RESULT_CAP_BYTES) {
3002
+ const room = EXEC_RESULT_CAP_BYTES - acc[which].length;
3003
+ acc[which] += data.length > room ? data.slice(0, room) : data;
3004
+ }
3005
+ try {
3006
+ onChunk(which, data);
3007
+ }
3008
+ catch {
3009
+ // frame capture must never break the exec itself
3010
+ }
3011
+ };
3012
+ child.stdout?.on("data", (c) => drain("stdout", c));
3013
+ child.stderr?.on("data", (c) => drain("stderr", c));
3014
+ let settled = false;
3015
+ const finish = (exitCode) => {
3016
+ if (settled)
3017
+ return;
3018
+ settled = true;
3019
+ // Flush any multi-byte tail the decoders are still holding.
3020
+ acc.stdout += decoders.stdout.decode();
3021
+ acc.stderr += decoders.stderr.decode();
3022
+ resolve({ stdout: acc.stdout, stderr: acc.stderr, exitCode });
3023
+ };
3024
+ // `close` (not `exit`) so both pipes are fully drained first.
3025
+ child.on("close", (code) => finish(code ?? 1));
3026
+ child.on("error", () => finish(1));
3027
+ });
3028
+ }
3029
+ async function pollCall(description, fn, opts) {
3030
+ const timeoutMs = opts?.timeoutMs ?? 30_000;
3031
+ const intervalMs = opts?.intervalMs ?? 1_000;
3032
+ const start = Date.now();
3033
+ // Reserve the wait's slot up front so it sorts at the poll's *start*,
3034
+ // ahead of the iteration events it nests (which record as the poll runs).
3035
+ const resv = reserveEvent();
3036
+ let attempts = 0;
3037
+ let value;
3038
+ let success = false;
3039
+ let predicateError;
3040
+ // Record all iterations normally. Falsy iterations get truncated
3041
+ // from the recorder so the timeline doesn't fill with polling
3042
+ // noise; the LAST iteration's events stay, then get marked as
3043
+ // children of the wait event so the UI can render them nested.
3044
+ const beforePollIdx = recorderEventCount();
3045
+ let lastIterStartIdx = beforePollIdx;
3046
+ let keptIterStartIdx = beforePollIdx;
3047
+ while (Date.now() - start < timeoutMs) {
3048
+ attempts += 1;
3049
+ lastIterStartIdx = recorderEventCount();
3050
+ try {
3051
+ const v = await fn();
3052
+ if (v !== null && v !== undefined && v !== false) {
3053
+ value = v;
3054
+ success = true;
3055
+ keptIterStartIdx = lastIterStartIdx;
3056
+ break;
3057
+ }
3058
+ }
3059
+ catch (err) {
3060
+ predicateError = err;
3061
+ break;
3062
+ }
3063
+ // Failed iteration — drop the events it emitted.
3064
+ recorderTruncate(lastIterStartIdx);
3065
+ if (Date.now() - start + intervalMs > timeoutMs)
3066
+ break;
3067
+ await new Promise((r) => setTimeout(r, intervalMs));
3068
+ }
3069
+ if (!success) {
3070
+ // Timeout or predicate error: drop every attempt's events. The
3071
+ // wait event we emit below is the only trace.
3072
+ recorderTruncate(beforePollIdx);
3073
+ }
3074
+ const errMsg = predicateError !== undefined
3075
+ ? (predicateError?.message ?? String(predicateError))
3076
+ : success
3077
+ ? undefined
3078
+ : `timed out after ${timeoutMs}ms`;
3079
+ const seq = recordWait({
3080
+ description,
3081
+ attempts,
3082
+ durationMs: Date.now() - start,
3083
+ passed: success,
3084
+ ...(errMsg !== undefined ? { error: errMsg } : {}),
3085
+ }, resv);
3086
+ if (success && seq !== undefined) {
3087
+ // Group the kept iteration's events under the wait so the UI can
3088
+ // render them inside the wait card. The wait event itself is the
3089
+ // very last entry; markChildren skips it via the seq match.
3090
+ recorderMarkChildren(keptIterStartIdx, seq);
3091
+ }
3092
+ if (predicateError !== undefined)
3093
+ throw predicateError;
3094
+ if (success) {
3095
+ return wrap(value, seq);
3096
+ }
3097
+ throw new Error(`poll ${JSON.stringify(description)} timed out after ${timeoutMs}ms (${attempts} attempts)`);
3098
+ }
3099
+ /**
3100
+ * Tee `console.*` output into `chunks` for the duration of a test/eval.
3101
+ * Bun's console writes through its own native sink, NOT
3102
+ * `process.stdout.write`, so patching the streams alone misses every
3103
+ * `console.log` the test makes — the captured `log` came back empty.
3104
+ * The original method still runs, so the daemon journal keeps the line.
3105
+ * Returns a restore function for the caller's `finally`.
3106
+ */
3107
+ function captureConsole(chunks) {
3108
+ const methods = ["log", "info", "warn", "error", "debug"];
3109
+ const orig = new Map();
3110
+ for (const m of methods) {
3111
+ const fn = console[m].bind(console);
3112
+ orig.set(m, console[m]);
3113
+ console[m] = (...a) => {
3114
+ try {
3115
+ chunks.push(a.map((x) => (typeof x === "string" ? x : Bun.inspect(x))).join(" ") + "\n");
3116
+ }
3117
+ catch {
3118
+ // capture must never break the test
3119
+ }
3120
+ fn(...a);
3121
+ };
3122
+ }
3123
+ return () => {
3124
+ for (const m of methods)
3125
+ console[m] = orig.get(m);
3126
+ };
3127
+ }
3128
+ async function runOne(testCase) {
3129
+ const start = Date.now();
3130
+ const chunks = [];
3131
+ const origStdout = process.stdout.write.bind(process.stdout);
3132
+ const origStderr = process.stderr.write.bind(process.stderr);
3133
+ const capture = (s) => {
3134
+ chunks.push(typeof s === "string" ? s : Buffer.from(s).toString("utf8"));
3135
+ return true;
3136
+ };
3137
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3138
+ process.stdout.write = capture;
3139
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3140
+ process.stderr.write = capture;
3141
+ const restoreConsole = captureConsole(chunks);
3142
+ // Terminal sessions: each `ctx.exec` / `ctx.terminal(...)` /
3143
+ // `ctx.openTerminal(...)` call opens its own session (asciicast frames
3144
+ // live on the record; the inline TestEvent just carries metadata + a
3145
+ // sessionId pointer). Drained at the end of the test and shipped on
3146
+ // RunResult.terminalSessions.
3147
+ //
3148
+ // Unlike browsers, we don't auto-close terminals at test end. A
3149
+ // `docker exec` subprocess is cheap to keep alive (a few KB), and
3150
+ // Freestyle's snapshot captures it cleanly along with the container
3151
+ // — so leaving it running between tests doesn't leak in any
3152
+ // meaningful sense. Auto-closing only added a noisy `close` step
3153
+ // at the end of every test that used `openTerminal`.
3154
+ const terminalSessions = [];
3155
+ // Wrap exec so each call shows up in the event log alongside its
3156
+ // result. We do this here (not on `execInService` itself) so the
3157
+ // bootstrap path stays uninstrumented. Each call also captures the
3158
+ // full CLI run as an asciicast — one exec step = one run = one
3159
+ // recording: output chunks are timestamped as they stream in, so slow
3160
+ // or animated output replays with real timing in the web UI. The
3161
+ // frames are presentation-only; the ExecResult (and any assertions on
3162
+ // it) still sees the plain separated stdout/stderr.
3163
+ const recordedExec = async (service, command, opts) => {
3164
+ const t = Date.now();
3165
+ const cwd = opts?.cwd;
3166
+ const resv = reserveEvent();
3167
+ const session = newTerminalSession(start, service, command, EXEC_CAST_COLS, EXEC_CAST_ROWS, testCase.id);
3168
+ terminalSessions.push(session.record);
3169
+ // Synthetic prompt frame so the replay is self-describing — the
3170
+ // program's own output starts on the next line, like a real shell. A
3171
+ // working directory rides in the prompt sigil (`svc:/dir $`) the way a
3172
+ // real shell prompt shows it, so the recording stays self-describing.
3173
+ const sigil = cwd ? `${service}:${cwd}` : service;
3174
+ session.pushFrame(0, `\x1b[32m${sigil} $\x1b[0m \x1b[1m${command}\x1b[0m\r\n`);
3175
+ let frameBytes = 0;
3176
+ let frameCapped = false;
3177
+ const res = await execInServiceStreaming(service, command, (_stream, data) => {
3178
+ if (frameCapped)
3179
+ return;
3180
+ if (frameBytes + data.length > EXEC_FRAME_CAP_BYTES) {
3181
+ frameCapped = true;
3182
+ session.pushFrame((Date.now() - t) / 1000, "\r\n\x1b[2m[spectest: recording truncated — output exceeded the cast cap]\x1b[0m\r\n");
3183
+ return;
3184
+ }
3185
+ frameBytes += data.length;
3186
+ // Pipes deliver bare `\n`; a terminal renderer needs `\r\n` or
3187
+ // every line starts at the previous line's end column
3188
+ // (stair-stepping). PTY output is ONLCR-cooked by the kernel —
3189
+ // pipe output is not, so cook it here. Normalising existing
3190
+ // `\r\n` too keeps a CR|LF split across chunk boundaries
3191
+ // harmless (`\r\r\n` renders identically).
3192
+ session.pushFrame((Date.now() - t) / 1000, data.replace(/\r?\n/g, "\r\n"));
3193
+ }, cwd);
3194
+ session.markClosed();
3195
+ const stdout = truncateUtf8(res.stdout);
3196
+ const stderr = truncateUtf8(res.stderr);
3197
+ const seq = recordExec({
3198
+ service,
3199
+ command,
3200
+ cwd,
3201
+ exitCode: res.exitCode,
3202
+ stdout: stdout.value,
3203
+ stdoutTruncated: stdout.truncated,
3204
+ stderr: stderr.value,
3205
+ stderrTruncated: stderr.truncated,
3206
+ durationMs: Date.now() - t,
3207
+ sessionId: session.record.sessionId,
3208
+ }, resv);
3209
+ return wrap(res, seq);
3210
+ };
3211
+ // One-shot: open a terminal with `command` as the entrypoint, wait
3212
+ // for it to exit, close, and return the existing TerminalResult
3213
+ // shape. The asciicast and one TerminalEvent line up exactly with
3214
+ // the pre-interactive implementation, just routed through the new
3215
+ // factory.
3216
+ const recordedTerminal = async (service, command, opts) => {
3217
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_TEST_TIMEOUT_MS;
3218
+ const startedAt = Date.now();
3219
+ const resv = reserveEvent();
3220
+ const term = await openInstrumentedTerminal(service, { ...opts, command, timeoutMs }, start, terminalSessions, false, // one-shot doesn't emit per-op step events
3221
+ testCase.id);
3222
+ // `term.exited` resolves to a wrapped result; this one-shot path needs
3223
+ // the plain `exitCode` number, so `.unwrap()` the result first.
3224
+ const { exitCode } = (await term.exited).unwrap();
3225
+ await term.close();
3226
+ const output = term.rawOutput();
3227
+ const preview = truncateUtf8(output);
3228
+ const seq = recordTerminal({
3229
+ service,
3230
+ command,
3231
+ exitCode,
3232
+ durationMs: Date.now() - startedAt,
3233
+ sessionId: term.sessionId,
3234
+ cols: term.cols,
3235
+ rows: term.rows,
3236
+ outputPreview: preview.value,
3237
+ outputTruncated: preview.truncated,
3238
+ }, resv);
3239
+ const result = {
3240
+ output,
3241
+ exitCode,
3242
+ durationMs: Date.now() - startedAt,
3243
+ sessionId: term.sessionId,
3244
+ };
3245
+ return wrap(result, seq);
3246
+ };
3247
+ // Long-lived: open an interactive terminal. Each method on the
3248
+ // returned Terminal records a `terminal-step` event tied back to
3249
+ // this session id; the per-op screen previews are written into the
3250
+ // event so the UI can show "what the user saw after sendLine 'ls'".
3251
+ const recordedOpenTerminal = async (service, opts) => {
3252
+ const term = await openInstrumentedTerminal(service, opts, start, terminalSessions, true, testCase.id);
3253
+ // Emit a one-time `terminal` event so the session shows up in the
3254
+ // sidebar even before any step lands. `exitCode` is filled in by
3255
+ // the eventual `exit`/`close` step event; the inline summary here
3256
+ // uses -1 as a sentinel until then.
3257
+ const preview = truncateUtf8(term.rawOutput());
3258
+ recordTerminal({
3259
+ service,
3260
+ command: opts?.command ?? "(interactive)",
3261
+ exitCode: -1,
3262
+ durationMs: 0,
3263
+ sessionId: term.sessionId,
3264
+ cols: term.cols,
3265
+ rows: term.rows,
3266
+ outputPreview: preview.value,
3267
+ outputTruncated: preview.truncated,
3268
+ });
3269
+ return term;
3270
+ };
3271
+ startRecording();
3272
+ const restoreFetch = installFetchWrapper();
3273
+ // Look up the parent's stored return value (if any). The parent ran in
3274
+ // an ancestor fork; its TEST_DATA entry travels with the snapshot.
3275
+ const parentId = testCase.dependsOn?.id;
3276
+ const parent = parentId !== undefined ? TEST_DATA.get(parentId) : undefined;
3277
+ // Browser/mobile sessions are PERSISTENT: `ctx.browser()` acquires THE
3278
+ // shared desktop browser and `ctx.mobile(app)` the one session for that
3279
+ // app (browser.ts's module-scoped registry, which forks with the
3280
+ // snapshot like fake state). At test end we DETACH — final rrweb drain,
3281
+ // stop writing to this test's recorder — but deliberately keep the
3282
+ // Chromium alive so the post-test snapshot captures it and dependsOn
3283
+ // children resume the live page (cookies, localStorage, signed-in SPA
3284
+ // state) instead of re-navigating. Each test still gets its own session
3285
+ // record (attach re-arms rrweb with a fresh full snapshot, so replays
3286
+ // stay per-case self-contained); records flow back to the control plane
3287
+ // on RunResult.browserSessions and are archived to S3 as the case's
3288
+ // replay bundle. Within one test repeated ctx.browser()/ctx.mobile(app)
3289
+ // calls return the same handle (memoized below) so one test = one
3290
+ // session per device. An explicit `.close()` destroys the shared
3291
+ // instance — the memo is cleared so a later call starts fresh.
3292
+ const browserDetaches = [];
3293
+ const sessions = [];
3294
+ let sharedBrowser = null;
3295
+ const sharedMobiles = new Map();
3296
+ const trackedOpenBrowser = async (opts) => {
3297
+ if (sharedBrowser)
3298
+ return sharedBrowser;
3299
+ const session = newBrowserSession(start, testCase.id);
3300
+ sessions.push(session);
3301
+ const { browser, attached, detach } = await acquirePersistentBrowser({
3302
+ ...(opts ?? {}),
3303
+ recorder: session.recorder,
3304
+ });
3305
+ if (browser.safeAreaInsets) {
3306
+ session.record.safeAreaInsets = browser.safeAreaInsets;
3307
+ }
3308
+ // An attached session starts mid-page (no navigate event will fire) —
3309
+ // stamp the inherited URL so the dashboard can still label the replay.
3310
+ if (attached && session.record.initialUrl === undefined) {
3311
+ session.record.initialUrl = browser.url();
3312
+ }
3313
+ browserDetaches.push(async () => {
3314
+ await detach();
3315
+ session.markClosed();
3316
+ });
3317
+ const innerClose = browser.close.bind(browser);
3318
+ browser.close = async () => {
3319
+ await innerClose();
3320
+ if (sharedBrowser === browser)
3321
+ sharedBrowser = null;
3322
+ };
3323
+ sharedBrowser = browser;
3324
+ return browser;
3325
+ };
3326
+ const trackedOpenMobile = async (app) => {
3327
+ if (!isMobileApp(app)) {
3328
+ throw new Error("ctx.mobile(app): pass a mobile-app handle from ctx.svc, e.g. ctx.mobile(ctx.svc.app) for a service declared with expo().");
3329
+ }
3330
+ const existing = sharedMobiles.get(app.url);
3331
+ if (existing)
3332
+ return existing;
3333
+ const session = newBrowserSession(start, testCase.id, "mobile");
3334
+ sessions.push(session);
3335
+ const { mobile, attached, detach, safeAreaInsets } = await openPersistentMobile({
3336
+ url: app.url,
3337
+ recorder: session.recorder,
3338
+ initScript: app.initScript,
3339
+ });
3340
+ if (safeAreaInsets)
3341
+ session.record.safeAreaInsets = safeAreaInsets;
3342
+ if (attached && session.record.initialUrl === undefined) {
3343
+ session.record.initialUrl = mobile.url();
3344
+ }
3345
+ browserDetaches.push(async () => {
3346
+ await detach();
3347
+ session.markClosed();
3348
+ });
3349
+ const innerClose = mobile.close.bind(mobile);
3350
+ mobile.close = async () => {
3351
+ await innerClose();
3352
+ if (sharedMobiles.get(app.url) === mobile)
3353
+ sharedMobiles.delete(app.url);
3354
+ };
3355
+ sharedMobiles.set(app.url, mobile);
3356
+ return mobile;
3357
+ };
3358
+ // Build convenience handles (e.g. ctx.svc.db.client) from the loaded
3359
+ // project. Done before installing the timeout so a slow client factory
3360
+ // surfaces as a real error rather than getting attributed to the test.
3361
+ const svc = await buildServiceHandles(requireLoaded().project.environment);
3362
+ const fakes = await buildFakeHandles();
3363
+ const ctx = {
3364
+ // installFetchWrapper just swapped globalThis.fetch for the wrapped
3365
+ // version, so capturing it here gets us instrumentation on ctx.fetch
3366
+ // for free. The recorder is active for the test, so responses come
3367
+ // back wrapped — hence the SpectestFetch type.
3368
+ fetch: globalThis.fetch,
3369
+ // exec/terminal/poll wrap their results at runtime (the recorder is
3370
+ // active), so the ctx interface types them wrapped — same bridge as
3371
+ // `fetch` above. The impls' own return types stay raw.
3372
+ exec: recordedExec,
3373
+ terminal: recordedTerminal,
3374
+ openTerminal: recordedOpenTerminal,
3375
+ browser: trackedOpenBrowser,
3376
+ mobile: trackedOpenMobile,
3377
+ testName: testCase.name,
3378
+ parent,
3379
+ svc,
3380
+ fakes,
3381
+ poll: pollCall,
3382
+ dnsName: registerDnsName,
3383
+ startService: startRuntimeService,
3384
+ stopService: stopRuntimeService,
3385
+ };
3386
+ const timeoutMs = testCase.timeoutMs ?? DEFAULT_TEST_TIMEOUT_MS;
3387
+ let timer;
3388
+ const timedOut = new Promise((_, reject) => {
3389
+ timer = setTimeout(() => reject(new Error(`test timed out after ${timeoutMs}ms`)), timeoutMs);
3390
+ });
3391
+ // Result events are gathered inside finally (after the final browser
3392
+ // drains) so we hoist these out of the try/catch.
3393
+ let outcome;
3394
+ try {
3395
+ const value = await Promise.race([
3396
+ Promise.resolve(testCase.run(ctx)),
3397
+ timedOut,
3398
+ ]);
3399
+ // Stash the return value so child cases — which fork from the snapshot
3400
+ // we're about to capture — can read it off ctx.parent.
3401
+ TEST_DATA.set(testCase.id, value);
3402
+ outcome = { status: "passed" };
3403
+ }
3404
+ catch (err) {
3405
+ const e = err;
3406
+ outcome = {
3407
+ status: "failed",
3408
+ error: { message: e.message ?? String(err), stack: e.stack },
3409
+ };
3410
+ }
3411
+ finally {
3412
+ if (timer)
3413
+ clearTimeout(timer);
3414
+ restoreFetch();
3415
+ restoreConsole();
3416
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3417
+ process.stdout.write = origStdout;
3418
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3419
+ process.stderr.write = origStderr;
3420
+ // Detach every browser/mobile session: final rrweb drain (must be
3421
+ // awaited before collecting session records), then stop writing to
3422
+ // this test's recorder. The Chromium itself deliberately stays alive
3423
+ // — it's part of the state the post-test snapshot captures for
3424
+ // dependsOn children (see the acquire comment above).
3425
+ for (const detach of browserDetaches) {
3426
+ try {
3427
+ await detach();
3428
+ }
3429
+ catch {
3430
+ /* ignore */
3431
+ }
3432
+ }
3433
+ for (const s of sessions)
3434
+ s.markClosed();
3435
+ }
3436
+ const durationMs = Date.now() - start;
3437
+ // Capture each service's log delta (the lines THIS case added beyond its
3438
+ // ancestors) on every case, pass or fail. Runs after the duration clock
3439
+ // stops so the `docker logs` round trips aren't billed to the test.
3440
+ // Advancing the markers here — before /run returns and the control plane
3441
+ // snapshots the fork — is what lets children tile their own deltas on
3442
+ // seamlessly. Shipped to S3: the dashboard reconstructs the full
3443
+ // cumulative log for a branch, the CLI failure post-mortem shows the
3444
+ // failing case's own delta.
3445
+ let serviceLogDeltas = [];
3446
+ try {
3447
+ serviceLogDeltas = await captureServiceLogDeltas();
3448
+ }
3449
+ catch (err) {
3450
+ // eslint-disable-next-line no-console
3451
+ console.warn("[service-logs] delta capture failed:", err);
3452
+ }
3453
+ const events = stopRecording();
3454
+ // Drop sessions whose linking event didn't survive — an exec/terminal
3455
+ // inside a failed `ctx.poll` iteration has its events removed by
3456
+ // recorderTruncate, so its recording would be an unreachable orphan in
3457
+ // the UI (and a polled exec would ship one dead cast per attempt).
3458
+ const referencedSessions = new Set();
3459
+ for (const ev of events) {
3460
+ const sid = ev.sessionId;
3461
+ if (typeof sid === "string")
3462
+ referencedSessions.add(sid);
3463
+ }
3464
+ return {
3465
+ status: outcome.status,
3466
+ durationMs,
3467
+ log: chunks.join(""),
3468
+ events,
3469
+ browserSessions: sessions.map((s) => s.record),
3470
+ terminalSessions: terminalSessions.filter((s) => referencedSessions.has(s.sessionId)),
3471
+ serviceLogDeltas,
3472
+ error: outcome.error,
3473
+ };
3474
+ }
3475
+ const EVAL_DIR = path.join(APP_DIR, ".spectest-eval");
3476
+ // Persistent state across eval calls. Mutated by snippets via the
3477
+ // `state` global; survives until the daemon process restarts.
3478
+ const EVAL_STATE = {};
3479
+ // Transpiler instance reused for `scanImports`. We don't transpile the
3480
+ // user code — Bun runs the .ts file directly — but scanImports gives us
3481
+ // the imports so we can auto-install missing deps.
3482
+ const SCAN_TRANSPILER = new Bun.Transpiler({ loader: "ts" });
3483
+ function safeSerialize(v) {
3484
+ if (v === undefined)
3485
+ return undefined;
3486
+ try {
3487
+ return JSON.parse(JSON.stringify(v));
3488
+ }
3489
+ catch {
3490
+ return String(v);
3491
+ }
3492
+ }
3493
+ /** Top-level package name from an import specifier. */
3494
+ function packageName(spec) {
3495
+ if (spec.startsWith("@")) {
3496
+ return spec.split("/").slice(0, 2).join("/");
3497
+ }
3498
+ return spec.split("/")[0];
3499
+ }
3500
+ /**
3501
+ * Scan the snippet's imports and `bun add` anything that doesn't already
3502
+ * resolve. Skips relative paths, absolute paths, `node:`/`bun:` built-ins,
3503
+ * and HTTP(S)/file: URLs.
3504
+ */
3505
+ async function ensureDeps(code) {
3506
+ let scanned;
3507
+ try {
3508
+ scanned = SCAN_TRANSPILER.scanImports(code);
3509
+ }
3510
+ catch {
3511
+ // Invalid syntax — let the import call surface the real error.
3512
+ return [];
3513
+ }
3514
+ const seen = new Set();
3515
+ const missing = [];
3516
+ for (const imp of scanned) {
3517
+ const p = imp.path;
3518
+ if (p.startsWith(".") ||
3519
+ p.startsWith("/") ||
3520
+ p.startsWith("node:") ||
3521
+ p.startsWith("bun:") ||
3522
+ p.startsWith("http:") ||
3523
+ p.startsWith("https:") ||
3524
+ p.startsWith("file:")) {
3525
+ continue;
3526
+ }
3527
+ const pkg = packageName(p);
3528
+ if (seen.has(pkg))
3529
+ continue;
3530
+ seen.add(pkg);
3531
+ try {
3532
+ Bun.resolveSync(p, APP_DIR);
3533
+ }
3534
+ catch {
3535
+ missing.push(pkg);
3536
+ }
3537
+ }
3538
+ if (missing.length === 0)
3539
+ return [];
3540
+ await new Promise((resolve, reject) => {
3541
+ execFile("/usr/local/bin/bun", ["add", ...missing], { cwd: APP_DIR, maxBuffer: 16 * 1024 * 1024 }, (err, stdout, stderr) => {
3542
+ if (err) {
3543
+ reject(new Error(`bun add ${missing.join(" ")} failed:\n${String(stderr).trim()}\n${String(stdout).trim()}`));
3544
+ }
3545
+ else {
3546
+ resolve();
3547
+ }
3548
+ });
3549
+ });
3550
+ return missing;
3551
+ }
3552
+ async function evalCode(code, secrets) {
3553
+ const start = Date.now();
3554
+ // Eval-scoped secret channel for record-mode fakes — set before the
3555
+ // snippet runs, cleared in the `finally` below so a secret never
3556
+ // persists into daemon memory (and thus into a forkable snapshot) past
3557
+ // the eval that supplied it. See record-secrets.ts.
3558
+ setRecordSecrets(secrets);
3559
+ const chunks = [];
3560
+ const origStdout = process.stdout.write.bind(process.stdout);
3561
+ const origStderr = process.stderr.write.bind(process.stderr);
3562
+ const capture = (s) => {
3563
+ chunks.push(typeof s === "string" ? s : Buffer.from(s).toString("utf8"));
3564
+ return true;
3565
+ };
3566
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3567
+ process.stdout.write = capture;
3568
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3569
+ process.stderr.write = capture;
3570
+ // Bun's console.* bypasses process.stdout.write — tee it too.
3571
+ const restoreConsole = captureConsole(chunks);
3572
+ // Wrap fetch for the snippet's duration so `ctx.fetch` returns a wrapped
3573
+ // Response just like in a test (no recorder here, so no provenance — but the
3574
+ // wrapped type is honest at runtime). Restored in the `finally` below.
3575
+ const restoreFetch = installFetchWrapper();
3576
+ // Same persistent acquire/detach as a test run (see runOne): the browser
3577
+ // survives the eval, so successive `spectest env eval` calls continue one
3578
+ // live session — and a snapshot taken afterwards carries it.
3579
+ const browserDetaches = [];
3580
+ const sessions = [];
3581
+ // Eval-only artifact sink — wiring it here (and nowhere in runOne) is
3582
+ // what gates screenshot() to eval context.
3583
+ const artifactCollector = newArtifactCollector();
3584
+ let sharedBrowser = null;
3585
+ const sharedMobiles = new Map();
3586
+ const trackedOpenBrowser = async (opts) => {
3587
+ if (sharedBrowser)
3588
+ return sharedBrowser;
3589
+ const session = newBrowserSession(start, "eval", "browser", artifactCollector);
3590
+ sessions.push(session);
3591
+ const { browser, attached, detach } = await acquirePersistentBrowser({
3592
+ ...(opts ?? {}),
3593
+ recorder: session.recorder,
3594
+ });
3595
+ if (browser.safeAreaInsets) {
3596
+ session.record.safeAreaInsets = browser.safeAreaInsets;
3597
+ }
3598
+ if (attached && session.record.initialUrl === undefined) {
3599
+ session.record.initialUrl = browser.url();
3600
+ }
3601
+ browserDetaches.push(async () => {
3602
+ await detach();
3603
+ session.markClosed();
3604
+ });
3605
+ const innerClose = browser.close.bind(browser);
3606
+ browser.close = async () => {
3607
+ await innerClose();
3608
+ if (sharedBrowser === browser)
3609
+ sharedBrowser = null;
3610
+ };
3611
+ sharedBrowser = browser;
3612
+ return browser;
3613
+ };
3614
+ const trackedOpenMobile = async (app) => {
3615
+ if (!isMobileApp(app)) {
3616
+ throw new Error("ctx.mobile(app): pass a mobile-app handle from ctx.svc, e.g. ctx.mobile(ctx.svc.app) for a service declared with expo().");
3617
+ }
3618
+ const existing = sharedMobiles.get(app.url);
3619
+ if (existing)
3620
+ return existing;
3621
+ const session = newBrowserSession(start, "eval", "mobile", artifactCollector);
3622
+ sessions.push(session);
3623
+ const { mobile, attached, detach, safeAreaInsets } = await openPersistentMobile({
3624
+ url: app.url,
3625
+ recorder: session.recorder,
3626
+ initScript: app.initScript,
3627
+ });
3628
+ if (safeAreaInsets)
3629
+ session.record.safeAreaInsets = safeAreaInsets;
3630
+ if (attached && session.record.initialUrl === undefined) {
3631
+ session.record.initialUrl = mobile.url();
3632
+ }
3633
+ browserDetaches.push(async () => {
3634
+ await detach();
3635
+ session.markClosed();
3636
+ });
3637
+ const innerClose = mobile.close.bind(mobile);
3638
+ mobile.close = async () => {
3639
+ await innerClose();
3640
+ if (sharedMobiles.get(app.url) === mobile)
3641
+ sharedMobiles.delete(app.url);
3642
+ };
3643
+ sharedMobiles.set(app.url, mobile);
3644
+ return mobile;
3645
+ };
3646
+ // Terminal sessions — same shape as runOne, but eval has no active
3647
+ // recorder so we don't emit inline events; the asciicast frames
3648
+ // still ship back on EvalResult.terminalSessions and the web UI
3649
+ // renders the player.
3650
+ const terminalSessions = [];
3651
+ const evalTerminal = async (service, command, opts) => {
3652
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_TEST_TIMEOUT_MS;
3653
+ const startedAt = Date.now();
3654
+ const term = await openInstrumentedTerminal(service, { ...opts, command, timeoutMs }, start, terminalSessions, false, "eval");
3655
+ const { exitCode } = (await term.exited).unwrap();
3656
+ await term.close();
3657
+ return {
3658
+ output: term.rawOutput(),
3659
+ exitCode,
3660
+ durationMs: Date.now() - startedAt,
3661
+ sessionId: term.sessionId,
3662
+ };
3663
+ };
3664
+ const evalOpenTerminal = async (service, opts) => {
3665
+ return openInstrumentedTerminal(service, opts, start, terminalSessions, false, "eval");
3666
+ };
3667
+ // Convenience handles are best-effort for eval — if the project isn't
3668
+ // loaded yet, fall back to an empty map so quick `await fetch(...)`
3669
+ // snippets don't require a /load round-trip first.
3670
+ const svc = loaded
3671
+ ? await buildServiceHandles(loaded.project.environment)
3672
+ : {};
3673
+ const fakes = loaded ? await buildFakeHandles() : {};
3674
+ const ctx = {
3675
+ // installFetchWrapper swapped globalThis.fetch above, so this captures the
3676
+ // wrapped version — eval results are wrapped just like in a test.
3677
+ fetch: globalThis.fetch,
3678
+ // Wraps its result the same way (no recorder under eval, so no provenance —
3679
+ // but the wrapped type is honest at runtime, so `.unwrap()` works).
3680
+ exec: execInServiceWrapped,
3681
+ terminal: evalTerminal,
3682
+ openTerminal: evalOpenTerminal,
3683
+ browser: trackedOpenBrowser,
3684
+ mobile: trackedOpenMobile,
3685
+ testName: "eval",
3686
+ parent: undefined,
3687
+ svc,
3688
+ fakes,
3689
+ poll: pollCall,
3690
+ dnsName: registerDnsName,
3691
+ startService: startRuntimeService,
3692
+ stopService: stopRuntimeService,
3693
+ };
3694
+ // Expose the test context, matchers, and persistent state as globals
3695
+ // so the snippet can use them without an explicit import. The user code
3696
+ // is real ESM, so `import { Client } from "pg"` and top-level `await`
3697
+ // work natively.
3698
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3699
+ const g = globalThis;
3700
+ g.ctx = ctx;
3701
+ g.expect = expect;
3702
+ g.expectRaw = expectRaw;
3703
+ g.assert = assert;
3704
+ g.state = EVAL_STATE;
3705
+ let installed = [];
3706
+ let filePath;
3707
+ let outcome;
3708
+ try {
3709
+ installed = await ensureDeps(code);
3710
+ await fs.mkdir(EVAL_DIR, { recursive: true });
3711
+ filePath = path.join(EVAL_DIR, `${randomUUID()}.ts`);
3712
+ await fs.writeFile(filePath, code);
3713
+ const mod = (await import(pathToFileURL(filePath).href));
3714
+ outcome = { ok: true, result: safeSerialize(mod.default) };
3715
+ }
3716
+ catch (err) {
3717
+ const e = err;
3718
+ outcome = {
3719
+ ok: false,
3720
+ error: { message: e.message ?? String(err), stack: e.stack },
3721
+ };
3722
+ }
3723
+ finally {
3724
+ clearRecordSecrets();
3725
+ restoreFetch();
3726
+ restoreConsole();
3727
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3728
+ process.stdout.write = origStdout;
3729
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
3730
+ process.stderr.write = origStderr;
3731
+ // Detach (final rrweb drain) — the browser itself stays alive; see
3732
+ // the acquire comment above.
3733
+ for (const detach of browserDetaches) {
3734
+ try {
3735
+ await detach();
3736
+ }
3737
+ catch {
3738
+ /* ignore */
3739
+ }
3740
+ }
3741
+ for (const s of sessions)
3742
+ s.markClosed();
3743
+ if (filePath) {
3744
+ fs.unlink(filePath).catch(() => {
3745
+ /* best-effort cleanup */
3746
+ });
3747
+ }
3748
+ }
3749
+ // `artifacts` ships on the error branch too — a screenshot captured
3750
+ // before a snippet crash is exactly the evidence the caller wants.
3751
+ return outcome.ok
3752
+ ? {
3753
+ ok: true,
3754
+ durationMs: Date.now() - start,
3755
+ log: chunks.join(""),
3756
+ installed,
3757
+ result: outcome.result,
3758
+ browserSessions: sessions.map((s) => s.record),
3759
+ terminalSessions,
3760
+ artifacts: artifactCollector.artifacts,
3761
+ }
3762
+ : {
3763
+ ok: false,
3764
+ durationMs: Date.now() - start,
3765
+ log: chunks.join(""),
3766
+ installed,
3767
+ browserSessions: sessions.map((s) => s.record),
3768
+ terminalSessions,
3769
+ artifacts: artifactCollector.artifacts,
3770
+ error: outcome.error,
3771
+ };
3772
+ }
3773
+ // ────────────────────────────────────────────────────────────────────────
3774
+ // Typecheck
3775
+ //
3776
+ // `tsc --noEmit` over the user's spectest/ code (APP_DIR — the only tree the
3777
+ // daemon imports; app-under-test source has its own toolchain). Kicked off in
3778
+ // the background at /load-tests so it never sits on the start/run critical
3779
+ // path; the control plane collects the result via POST /typecheck (which
3780
+ // awaits the in-flight run) and attaches it to the SuiteResult as an
3781
+ // advisory report. It must never gate a run: the runtime accepts patterns
3782
+ // the types reject (e.g. bare global `fetch` is monkey-patched to record,
3783
+ // but its *type* stays raw), so a type error is a strong hint, not proof.
3784
+ //
3785
+ // The compiler is the native tsc baked into the base snapshot at
3786
+ // /opt/spectest/typecheck (see base.rs BASE_SETUP_SH); a project that ships
3787
+ // its own `typescript` in spectest/package.json wins. A project tsconfig.json
3788
+ // wins over the generated one the same way.
3789
+ // ────────────────────────────────────────────────────────────────────────
3790
+ const TYPECHECK_DIR = "/opt/spectest/typecheck";
3791
+ /** Cap on errors shipped in the report; `totalErrors` carries the true count. */
3792
+ const TYPECHECK_ERROR_CAP = 50;
3793
+ let TYPECHECK = null;
3794
+ /** Fire-and-forget kickoff; the promise is parked for POST /typecheck. */
3795
+ function startTypecheck() {
3796
+ TYPECHECK = runTypecheck().catch((err) => ({
3797
+ status: "failed",
3798
+ errors: [],
3799
+ totalErrors: 0,
3800
+ durationMs: 0,
3801
+ detail: err instanceof Error ? err.message : String(err),
3802
+ }));
3803
+ // Parked promises must never surface as unhandled rejections.
3804
+ TYPECHECK.catch(() => { });
3805
+ }
3806
+ /** The tsconfig used when the project doesn't ship its own. `lib` is left to
3807
+ * the target default (which includes DOM — browser.evaluate callbacks
3808
+ * reference `document`); `types` pulls Bun's ambient globals from the baked
3809
+ * install via `typeRoots` (the walk-up from APP_DIR never reaches it). */
3810
+ function generatedTsconfig() {
3811
+ return JSON.stringify({
3812
+ compilerOptions: {
3813
+ target: "esnext",
3814
+ module: "esnext",
3815
+ moduleResolution: "bundler",
3816
+ strict: true,
3817
+ noEmit: true,
3818
+ skipLibCheck: true,
3819
+ esModuleInterop: true,
3820
+ resolveJsonModule: true,
3821
+ jsx: "react-jsx",
3822
+ types: ["bun"],
3823
+ typeRoots: [
3824
+ path.join(APP_DIR, "node_modules", "@types"),
3825
+ path.join(TYPECHECK_DIR, "node_modules", "@types"),
3826
+ ],
3827
+ },
3828
+ include: ["**/*.ts", "**/*.tsx"],
3829
+ exclude: ["node_modules"],
3830
+ }, null, 2);
3831
+ }
3832
+ /** Parse `--pretty false` tsc output: `file(line,col): error TScode: msg`,
3833
+ * with indented elaboration lines folded into the preceding error. */
3834
+ function parseTscOutput(output) {
3835
+ const errors = [];
3836
+ let total = 0;
3837
+ let suppressed = 0;
3838
+ let last = null;
3839
+ for (const line of output.split("\n")) {
3840
+ const m = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/.exec(line);
3841
+ if (!m) {
3842
+ // Elaboration lines are indented; fold them into the last kept error.
3843
+ if (last && /^\s+\S/.test(line))
3844
+ last.message += `\n${line.trimEnd()}`;
3845
+ continue;
3846
+ }
3847
+ const [, rawFile, ln, col, code, message] = m;
3848
+ // Keep only diagnostics in the user's files. tsc runs with cwd=APP_DIR
3849
+ // (the daemon's cwd), so project files come out relative ("tests/x.ts")
3850
+ // and anything outside — the SDK at /opt/spectest/sdk, node_modules —
3851
+ // is `../…` or absolute. Errors in our own SDK are ours to fix, not the
3852
+ // user's to read.
3853
+ const abs = path.resolve(APP_DIR, rawFile);
3854
+ const inApp = abs.startsWith(APP_DIR + path.sep) && !abs.includes(`${path.sep}node_modules${path.sep}`);
3855
+ if (!inApp) {
3856
+ suppressed += 1;
3857
+ last = null;
3858
+ continue;
3859
+ }
3860
+ total += 1;
3861
+ const err = {
3862
+ file: path.relative(APP_DIR, abs),
3863
+ line: Number(ln),
3864
+ column: Number(col),
3865
+ code: code,
3866
+ message: message,
3867
+ };
3868
+ if (errors.length < TYPECHECK_ERROR_CAP) {
3869
+ errors.push(err);
3870
+ last = err;
3871
+ }
3872
+ else {
3873
+ last = null;
3874
+ }
3875
+ }
3876
+ return { errors, total, suppressed };
3877
+ }
3878
+ async function runTypecheck() {
3879
+ const started = Date.now();
3880
+ // The project's own typescript wins over the baked copy.
3881
+ const tsc = [
3882
+ path.join(APP_DIR, "node_modules", "typescript", "bin", "tsc"),
3883
+ path.join(TYPECHECK_DIR, "node_modules", "typescript", "bin", "tsc"),
3884
+ ].find((p) => existsSync(p));
3885
+ if (!tsc) {
3886
+ return {
3887
+ status: "skipped",
3888
+ errors: [],
3889
+ totalErrors: 0,
3890
+ durationMs: 0,
3891
+ detail: "typescript is not installed (base image predates the baked typechecker)",
3892
+ };
3893
+ }
3894
+ let config = path.join(APP_DIR, "tsconfig.json");
3895
+ if (!existsSync(config)) {
3896
+ config = path.join(APP_DIR, ".spectest-tsconfig.json");
3897
+ await fs.writeFile(config, generatedTsconfig());
3898
+ }
3899
+ const res = await shx(process.execPath, [tsc, "--noEmit", "--pretty", "false", "-p", config], 120_000);
3900
+ const durationMs = Date.now() - started;
3901
+ if (res.code === 124) {
3902
+ return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: "typecheck timed out" };
3903
+ }
3904
+ const { errors, total, suppressed } = parseTscOutput(res.stdout + res.stderr);
3905
+ if (errors.length === 0) {
3906
+ // Exit 0 → clean. Non-zero with no *user-file* diagnostics is either
3907
+ // all-suppressed (still ok from the user's perspective) or a compiler
3908
+ // crash (config not found, OOM) — surface the latter.
3909
+ if (res.code !== 0 && suppressed === 0) {
3910
+ const tail = (res.stderr || res.stdout).trim().split("\n").slice(-5).join("\n");
3911
+ return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: tail || `tsc exited ${res.code}` };
3912
+ }
3913
+ if (suppressed > 0) {
3914
+ console.warn(`[typecheck] ${suppressed} diagnostic(s) outside the project suppressed`);
3915
+ }
3916
+ return { status: "ok", errors: [], totalErrors: 0, durationMs };
3917
+ }
3918
+ return { status: "errors", errors, totalErrors: total, durationMs };
3919
+ }
3920
+ // ────────────────────────────────────────────────────────────────────────
3921
+ // HTTP server
3922
+ // ────────────────────────────────────────────────────────────────────────
3923
+ async function readBody(req) {
3924
+ return new Promise((resolve, reject) => {
3925
+ const chunks = [];
3926
+ req.on("data", (c) => chunks.push(c));
3927
+ req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
3928
+ req.on("error", reject);
3929
+ });
3930
+ }
3931
+ function jsonResponse(res, status, body) {
3932
+ const payload = Buffer.from(JSON.stringify(body));
3933
+ res.writeHead(status, {
3934
+ "content-type": "application/json",
3935
+ "content-length": payload.length,
3936
+ });
3937
+ res.end(payload);
3938
+ }
3939
+ async function handle(req, res, state) {
3940
+ const url = req.url ?? "";
3941
+ const method = req.method ?? "GET";
3942
+ if (method === "GET" && url === "/health") {
3943
+ res.writeHead(200, { "content-type": "text/plain" });
3944
+ res.end("ok\n");
3945
+ return;
3946
+ }
3947
+ if (method === "GET" && url === "/progress") {
3948
+ // Live bootstrap progress, polled by the control plane during
3949
+ // /bootstrap and streamed into the test-run row. `{}` before the
3950
+ // first bootstrap() of this daemon.
3951
+ jsonResponse(res, 200, BOOTSTRAP_PROGRESS ?? {});
3952
+ return;
3953
+ }
3954
+ if (method === "POST" && url === "/load") {
3955
+ // Env only — services/fakes/setup. For the legacy single-file layout the
3956
+ // entry also defines the tests, so `cases` is populated here; for the
3957
+ // split layout `cases` is empty until /load-tests runs.
3958
+ const proj = await loadEnv();
3959
+ jsonResponse(res, 200, {
3960
+ environment: proj.environment,
3961
+ cases: casesMetadata(proj.tests),
3962
+ fakes: fakesSummary(proj),
3963
+ });
3964
+ return;
3965
+ }
3966
+ if (method === "POST" && url === "/load-tests") {
3967
+ // Import spectest/tests/** into the already-loaded env and return the
3968
+ // resulting catalogue. Called after the warm snapshot (cold path) or
3969
+ // against a freshly restored VM (warm path).
3970
+ await loadTests();
3971
+ // Background typecheck of the freshly-uploaded app tree — both paths
3972
+ // (cold and warm) funnel through here after the upload, so the check
3973
+ // always sees the current code. Never on the critical path: the reply
3974
+ // doesn't wait, POST /typecheck collects.
3975
+ startTypecheck();
3976
+ const l = requireLoaded();
3977
+ jsonResponse(res, 200, {
3978
+ environment: l.project.environment,
3979
+ cases: casesMetadata(l.project.tests),
3980
+ fakes: fakesSummary(l.project),
3981
+ });
3982
+ return;
3983
+ }
3984
+ if (method === "POST" && url === "/typecheck") {
3985
+ // Await the run kicked off at /load-tests (or start one on demand —
3986
+ // e.g. a legacy single-file project loaded before this daemon shipped
3987
+ // the kickoff, or a direct debug call).
3988
+ if (!TYPECHECK)
3989
+ startTypecheck();
3990
+ jsonResponse(res, 200, await TYPECHECK);
3991
+ return;
3992
+ }
3993
+ if (method === "POST" && url === "/unload") {
3994
+ loaded = null;
3995
+ jsonResponse(res, 200, { unloaded: true });
3996
+ return;
3997
+ }
3998
+ if (method === "POST" && url === "/reload") {
3999
+ // Full reload (debug aid): re-import the env, then the tests.
4000
+ await loadEnv();
4001
+ await loadTests();
4002
+ const l = requireLoaded();
4003
+ jsonResponse(res, 200, {
4004
+ environment: l.project.environment,
4005
+ cases: casesMetadata(l.project.tests),
4006
+ fakes: fakesSummary(l.project),
4007
+ });
4008
+ return;
4009
+ }
4010
+ if (method === "GET" && url === "/env-config") {
4011
+ const l = requireLoaded();
4012
+ jsonResponse(res, 200, l.project.environment);
4013
+ return;
4014
+ }
4015
+ if (method === "GET" && url === "/cases") {
4016
+ const l = requireLoaded();
4017
+ jsonResponse(res, 200, { cases: casesMetadata(l.project.tests) });
4018
+ return;
4019
+ }
4020
+ if (method === "GET" && url === "/record-secret-refs") {
4021
+ // Union of platform secret refs the loaded fakes declare (replayFake's
4022
+ // `secretRefs`). The control plane resolves these server-side and
4023
+ // pushes the values on the eval path only. Empty if nothing's loaded.
4024
+ const refs = new Set();
4025
+ for (const fake of FAKES.values()) {
4026
+ for (const ref of fake.def.secretRefs ?? [])
4027
+ refs.add(ref);
4028
+ }
4029
+ jsonResponse(res, 200, { refs: [...refs] });
4030
+ return;
4031
+ }
4032
+ if (method === "POST" && url === "/bootstrap") {
4033
+ if (state.inFlightBootstrap) {
4034
+ jsonResponse(res, 409, { error: "bootstrap already in progress" });
4035
+ return;
4036
+ }
4037
+ const job = bootstrap();
4038
+ state.inFlightBootstrap = job;
4039
+ try {
4040
+ const timings = await job;
4041
+ jsonResponse(res, 200, { ok: true, timings });
4042
+ }
4043
+ finally {
4044
+ state.inFlightBootstrap = null;
4045
+ }
4046
+ return;
4047
+ }
4048
+ if (method === "POST" && url === "/project-setup") {
4049
+ if (state.inFlightProjectSetup) {
4050
+ jsonResponse(res, 409, { error: "project-setup already in progress" });
4051
+ return;
4052
+ }
4053
+ const job = runProjectSetup();
4054
+ state.inFlightProjectSetup = job;
4055
+ try {
4056
+ const result = await job;
4057
+ jsonResponse(res, 200, result);
4058
+ }
4059
+ finally {
4060
+ state.inFlightProjectSetup = null;
4061
+ }
4062
+ return;
4063
+ }
4064
+ if (method === "POST" && url === "/eval") {
4065
+ if (state.inFlightTest) {
4066
+ jsonResponse(res, 409, { error: "a test or eval is already running" });
4067
+ return;
4068
+ }
4069
+ const body = await readBody(req);
4070
+ let parsed;
4071
+ try {
4072
+ parsed = JSON.parse(body || "{}");
4073
+ }
4074
+ catch {
4075
+ jsonResponse(res, 400, { error: "invalid JSON body" });
4076
+ return;
4077
+ }
4078
+ const code = parsed.code;
4079
+ if (typeof code !== "string" || code.length === 0) {
4080
+ jsonResponse(res, 400, { error: "code (string) is required" });
4081
+ return;
4082
+ }
4083
+ // `secrets` are eval-scoped: the control plane resolves the loaded
4084
+ // project's declared `replayFake` refs and pushes the values here on
4085
+ // the eval path only. Never present on the /run (test) path.
4086
+ const exec = evalCode(code, parsed.secrets);
4087
+ state.inFlightTest = exec;
4088
+ try {
4089
+ const result = await exec;
4090
+ jsonResponse(res, 200, result);
4091
+ }
4092
+ finally {
4093
+ state.inFlightTest = null;
4094
+ }
4095
+ return;
4096
+ }
4097
+ if (method === "POST" && url === "/capture-log-baseline") {
4098
+ // Snapshot each service's log output produced during env bring-up
4099
+ // (container startup + project setup), advancing the log markers so the
4100
+ // subsequent per-test deltas start AFTER setup. The control plane calls
4101
+ // this once, on the main env, before the first test runs — see
4102
+ // tests.rs::capture_log_baseline. Best-effort: on failure the setup lines
4103
+ // simply fold into the first test's delta as before.
4104
+ let serviceLogDeltas = [];
4105
+ try {
4106
+ serviceLogDeltas = await captureServiceLogDeltas();
4107
+ }
4108
+ catch (err) {
4109
+ // eslint-disable-next-line no-console
4110
+ console.warn("[service-logs] baseline capture failed:", err);
4111
+ }
4112
+ jsonResponse(res, 200, { serviceLogDeltas });
4113
+ return;
4114
+ }
4115
+ if (method === "POST" && url === "/run") {
4116
+ if (state.inFlightTest) {
4117
+ jsonResponse(res, 409, { error: "another test is already running" });
4118
+ return;
4119
+ }
4120
+ const body = await readBody(req);
4121
+ let parsed;
4122
+ try {
4123
+ parsed = JSON.parse(body || "{}");
4124
+ }
4125
+ catch {
4126
+ jsonResponse(res, 400, { error: "invalid JSON body" });
4127
+ return;
4128
+ }
4129
+ const caseId = parsed.caseId;
4130
+ if (!caseId) {
4131
+ jsonResponse(res, 400, { error: "caseId is required" });
4132
+ return;
4133
+ }
4134
+ const l = requireLoaded();
4135
+ const tc = l.byId.get(caseId);
4136
+ if (!tc) {
4137
+ jsonResponse(res, 404, { error: `unknown caseId: ${caseId}` });
4138
+ return;
4139
+ }
4140
+ const exec = runOne(tc);
4141
+ state.inFlightTest = exec;
4142
+ try {
4143
+ const result = await exec;
4144
+ // Recordings go out-of-band: encode + park the bundle, reply with a
4145
+ // size ref only (see the REPLAY_BUNDLES comment). A bundle-encoding
4146
+ // failure drops the recordings (warn) rather than failing the case —
4147
+ // and never falls back to inlining, which is exactly the >16 MB
4148
+ // response this path exists to avoid.
4149
+ const { browserSessions, ...wire } = result;
4150
+ let replay;
4151
+ if (browserSessions.length > 0) {
4152
+ try {
4153
+ const gz = encodeReplayBundle(caseId, browserSessions);
4154
+ stashReplayBundle(caseId, gz);
4155
+ replay = { bytes: gz.length };
4156
+ }
4157
+ catch (err) {
4158
+ // eslint-disable-next-line no-console
4159
+ console.warn("[replay] bundle encode failed; dropping recordings:", err);
4160
+ }
4161
+ }
4162
+ jsonResponse(res, 200, { ...wire, replay });
4163
+ }
4164
+ finally {
4165
+ state.inFlightTest = null;
4166
+ }
4167
+ return;
4168
+ }
4169
+ if (method === "POST" && url === "/replay-chunk") {
4170
+ // One chunk of a parked replay bundle (see REPLAY_BUNDLES). POST with
4171
+ // a JSON body — case ids are arbitrary user strings, and a body dodges
4172
+ // URL-encoding across both providers' daemon_http transports.
4173
+ const body = await readBody(req);
4174
+ let parsed;
4175
+ try {
4176
+ parsed = JSON.parse(body || "{}");
4177
+ }
4178
+ catch {
4179
+ jsonResponse(res, 400, { error: "invalid JSON body" });
4180
+ return;
4181
+ }
4182
+ if (!parsed.caseId) {
4183
+ jsonResponse(res, 400, { error: "caseId is required" });
4184
+ return;
4185
+ }
4186
+ const gz = REPLAY_BUNDLES.get(parsed.caseId);
4187
+ if (!gz) {
4188
+ jsonResponse(res, 404, { error: `no replay bundle for case: ${parsed.caseId}` });
4189
+ return;
4190
+ }
4191
+ jsonResponse(res, 200, replayChunk(gz, parsed.offset, parsed.limit));
4192
+ return;
4193
+ }
4194
+ jsonResponse(res, 404, { error: "not found" });
4195
+ }
4196
+ async function main() {
4197
+ const state = {
4198
+ inFlightTest: null,
4199
+ inFlightBootstrap: null,
4200
+ inFlightProjectSetup: null,
4201
+ };
4202
+ const server = http.createServer((req, res) => {
4203
+ handle(req, res, state).catch((err) => {
4204
+ const e = err;
4205
+ try {
4206
+ jsonResponse(res, 500, { error: e.message ?? String(err), stack: e.stack });
4207
+ }
4208
+ catch {
4209
+ // headers already sent or socket dead
4210
+ }
4211
+ });
4212
+ });
4213
+ const port = Number(process.env.SPECTEST_DAEMON_PORT ?? DEFAULT_PORT);
4214
+ server.listen(port, "0.0.0.0", () => {
4215
+ // eslint-disable-next-line no-console
4216
+ console.log(`spectest-daemon listening on :${port} (idle; awaiting POST /load)`);
4217
+ });
4218
+ }
4219
+ main().catch((err) => {
4220
+ // eslint-disable-next-line no-console
4221
+ console.error("spectest-daemon: fatal:", err);
4222
+ process.exit(1);
4223
+ });