@specific.dev/spectest 0.20.0 → 0.22.0

package/src/components/replayFake.ts

@@ -519,10 +519,9 @@ function resolveExternalA(host: string): Promise<string[]> {
     sock.on("message", (msg) => {
       try {
         const res = dnsPacket.decode(msg);
-        const ips = (res.answers ?? [])
-          .filter((a) => a.type === "A")
-          .map((a) => a.data as string)
-          .filter((d) => typeof d === "string" && d.length > 0);
+        const ips = (res.answers ?? []).flatMap((a) =>
+          a.type === "A" && typeof a.data === "string" && a.data.length > 0 ? [a.data] : [],
+        );
         finish(ips);
       } catch {
         finish([]);
@@ -766,7 +765,7 @@ export function replayFake(
     state.recorded.push(interaction);
     state.dirty = true;
 
-    return new Response(upstream.bytes, {
+    return new Response(upstream.bytes as unknown as BodyInit, {
       status: upstream.status,
       headers: new Headers(respHeaders),
     });

package/src/daemon.ts

@@ -57,6 +57,7 @@ import {
 import { wrap, wrapResponse } from "./inspect.js";
 import type { SpectestFetch, Wrapped, WrappedResponse } from "./inspect.js";
 import { clearRecordSecrets, setRecordSecrets } from "./record-secrets.js";
+import { encodeReplayBundle, replayChunk } from "./replay-bundle.js";
 
 import type {
   Browser,
@@ -750,7 +751,7 @@ const BUILD_DEDUP = new Map<
   { name: string; promise: Promise<{ tag: string; buildSteps?: BuildStep[] }> }
 >();
 
-function buildContentKey(image: { content: string; exclude?: string[] }): string {
+function buildContentKey(image: { content: string; exclude?: readonly string[] }): string {
   return new Bun.CryptoHasher("sha256")
     .update(image.content)
     .update("\0")
@@ -836,7 +837,7 @@ async function prepareServiceImage(
 
 async function buildServiceImage(
   name: string,
-  image: { content: string; exclude?: string[] },
+  image: { content: string; exclude?: readonly string[] },
   tag: string,
 ): Promise<{ tag: string; buildSteps?: BuildStep[] }> {
   let buildSteps: BuildStep[] | undefined;
@@ -2802,6 +2803,38 @@ interface RunResult {
   error?: { message: string; stack?: string };
 }
 
+// ────────────────────────────────────────────────────────────────────────
+// Replay bundles (rrweb sessions → gzipped side-channel, pulled in chunks)
+// ────────────────────────────────────────────────────────────────────────
+//
+// `RunResult.browserSessions` never rides the `/run` reply: the vm-agent
+// caps proxied daemon responses at 16 MB and a browser-heavy case's
+// recording is tens of MB of JSON (see replay-bundle.ts for the whole
+// story). Instead the /run handler encodes the sessions into a gzipped,
+// asset-deduplicated bundle, parks it here keyed by case id, and replies
+// with a tiny `replay: { bytes }` ref; the control plane then pulls the
+// bundle via `POST /replay-chunk` in base64 chunks sized under the cap,
+// before it tears the fork down.
+//
+// The map is module memory, so it FORKS with the snapshot (the post-test
+// snapshot is captured before the control plane fetches). The size cap
+// keeps a long handoff chain from accreting every ancestor's (gzipped)
+// bundle in guest RAM — entries older than the last few are always
+// already-fetched leftovers frozen into some snapshot.
+const REPLAY_BUNDLES = new Map<string, Buffer>();
+const REPLAY_BUNDLES_MAX = 8;
+
+function stashReplayBundle(caseId: string, gz: Buffer): void {
+  // Re-insert to refresh recency (Map iterates in insertion order).
+  REPLAY_BUNDLES.delete(caseId);
+  REPLAY_BUNDLES.set(caseId, gz);
+  while (REPLAY_BUNDLES.size > REPLAY_BUNDLES_MAX) {
+    const oldest = REPLAY_BUNDLES.keys().next().value;
+    if (oldest === undefined) break;
+    REPLAY_BUNDLES.delete(oldest);
+  }
+}
+
 // ────────────────────────────────────────────────────────────────────────
 // Cumulative service logs (per-case deltas → S3, reconstructed on the web)
 // ────────────────────────────────────────────────────────────────────────
@@ -2846,6 +2879,32 @@ interface ServiceLogDelta {
   stderr: string;
   stderrTruncated: boolean;
   stderrReset: boolean;
+  /** The container was no longer running ("exited"/"dead") when this delta
+   *  was captured. Absent while running or when inspect fails (e.g. the
+   *  container was removed). Shown next to the service on the dashboard's
+   *  Logs page. */
+  stopped?: boolean;
+  /** docker's recorded exit code for the stopped container. */
+  exitCode?: number;
+}
+
+/**
+ * Inspect a container's run state for {@link ServiceLogDelta}: `{}` while
+ * running (or when inspect fails — a removed container has no state left to
+ * report), `{ stopped, exitCode }` once it has exited/died.
+ */
+async function containerStopState(
+  name: string,
+): Promise<{ stopped?: boolean; exitCode?: number }> {
+  const r = await docker(
+    ["inspect", "-f", "{{.State.Status}} {{.State.ExitCode}}", name],
+    15_000,
+  );
+  if (r.code !== 0) return {};
+  const [status, codeStr] = r.stdout.trim().split(/\s+/);
+  if (status !== "exited" && status !== "dead") return {};
+  const exitCode = Number.parseInt(codeStr ?? "", 10);
+  return Number.isFinite(exitCode) ? { stopped: true, exitCode } : { stopped: true };
 }
 
 /** Keep the head and tail of `s`, eliding the middle when it exceeds
@@ -2924,7 +2983,10 @@ async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
   const services = [...byName.values()];
   return Promise.all(
     services.map(async (svc): Promise<ServiceLogDelta> => {
-      const r = await docker(["logs", "--timestamps", svc.name], 30_000);
+      const [r, stop] = await Promise.all([
+        docker(["logs", "--timestamps", svc.name], 30_000),
+        containerStopState(svc.name),
+      ]);
       if (r.code !== 0) {
         // Container gone/renamed — surface the CLI error, leave markers put.
         const err = capMiddle(r.stderr || r.stdout, LOG_DELTA_MAX_BYTES);
@@ -2936,6 +2998,7 @@ async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
           stderr: err.value,
           stderrTruncated: err.truncated,
           stderrReset: false,
+          ...stop,
         };
       }
       const outKey = `${svc.name}stdout`;
@@ -2952,6 +3015,7 @@ async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
         stderr: err.delta,
         stderrTruncated: err.truncated,
         stderrReset: err.reset,
+        ...stop,
       };
     }),
   );
@@ -3882,7 +3946,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     // An attached session starts mid-page (no navigate event will fire) —
     // stamp the inherited URL so the dashboard can still label the replay.
     if (attached && session.record.initialUrl === undefined) {
-      session.record.initialUrl = browser.url;
+      session.record.initialUrl = browser.url();
     }
     browserDetaches.push(async () => {
       await detach();
@@ -3912,7 +3976,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     });
     if (safeAreaInsets) session.record.safeAreaInsets = safeAreaInsets;
     if (attached && session.record.initialUrl === undefined) {
-      session.record.initialUrl = mobile.url;
+      session.record.initialUrl = mobile.url();
     }
     browserDetaches.push(async () => {
       await detach();
@@ -4204,7 +4268,7 @@ async function evalCode(
       session.record.safeAreaInsets = browser.safeAreaInsets;
     }
     if (attached && session.record.initialUrl === undefined) {
-      session.record.initialUrl = browser.url;
+      session.record.initialUrl = browser.url();
     }
     browserDetaches.push(async () => {
       await detach();
@@ -4234,7 +4298,7 @@ async function evalCode(
     });
     if (safeAreaInsets) session.record.safeAreaInsets = safeAreaInsets;
     if (attached && session.record.initialUrl === undefined) {
-      session.record.initialUrl = mobile.url;
+      session.record.initialUrl = mobile.url();
     }
     browserDetaches.push(async () => {
       await detach();
@@ -4400,6 +4464,185 @@ async function evalCode(
       };
 }
 
+// ────────────────────────────────────────────────────────────────────────
+// Typecheck
+//
+// `tsc --noEmit` over the user's spectest/ code (APP_DIR — the only tree the
+// daemon imports; app-under-test source has its own toolchain). Kicked off in
+// the background at /load-tests so it never sits on the start/run critical
+// path; the control plane collects the result via POST /typecheck (which
+// awaits the in-flight run) and attaches it to the SuiteResult as an
+// advisory report. It must never gate a run: the runtime accepts patterns
+// the types reject (e.g. bare global `fetch` is monkey-patched to record,
+// but its *type* stays raw), so a type error is a strong hint, not proof.
+//
+// The compiler is the native tsc baked into the base snapshot at
+// /opt/spectest/typecheck (see base.rs BASE_SETUP_SH); a project that ships
+// its own `typescript` in spectest/package.json wins. A project tsconfig.json
+// wins over the generated one the same way.
+// ────────────────────────────────────────────────────────────────────────
+
+const TYPECHECK_DIR = "/opt/spectest/typecheck";
+/** Cap on errors shipped in the report; `totalErrors` carries the true count. */
+const TYPECHECK_ERROR_CAP = 50;
+
+interface TypecheckError {
+  /** Path relative to the project's spectest/ dir (e.g. `tests/checkout.ts`). */
+  file: string;
+  line: number;
+  column: number;
+  /** `TS2345` etc. */
+  code: string;
+  message: string;
+}
+
+interface TypecheckReport {
+  status: "ok" | "errors" | "skipped" | "failed";
+  errors: TypecheckError[];
+  totalErrors: number;
+  durationMs: number;
+  /** Why the check was skipped / how it failed. */
+  detail?: string;
+}
+
+let TYPECHECK: Promise<TypecheckReport> | null = null;
+
+/** Fire-and-forget kickoff; the promise is parked for POST /typecheck. */
+function startTypecheck(): void {
+  TYPECHECK = runTypecheck().catch((err) => ({
+    status: "failed" as const,
+    errors: [],
+    totalErrors: 0,
+    durationMs: 0,
+    detail: err instanceof Error ? err.message : String(err),
+  }));
+  // Parked promises must never surface as unhandled rejections.
+  TYPECHECK.catch(() => {});
+}
+
+/** The tsconfig used when the project doesn't ship its own. `lib` is left to
+ *  the target default (which includes DOM — browser.evaluate callbacks
+ *  reference `document`); `types` pulls Bun's ambient globals from the baked
+ *  install via `typeRoots` (the walk-up from APP_DIR never reaches it). */
+function generatedTsconfig(): string {
+  return JSON.stringify(
+    {
+      compilerOptions: {
+        target: "esnext",
+        module: "esnext",
+        moduleResolution: "bundler",
+        strict: true,
+        noEmit: true,
+        skipLibCheck: true,
+        esModuleInterop: true,
+        resolveJsonModule: true,
+        jsx: "react-jsx",
+        types: ["bun"],
+        typeRoots: [
+          path.join(APP_DIR, "node_modules", "@types"),
+          path.join(TYPECHECK_DIR, "node_modules", "@types"),
+        ],
+      },
+      include: ["**/*.ts", "**/*.tsx"],
+      exclude: ["node_modules"],
+    },
+    null,
+    2,
+  );
+}
+
+/** Parse `--pretty false` tsc output: `file(line,col): error TScode: msg`,
+ *  with indented elaboration lines folded into the preceding error. */
+function parseTscOutput(output: string): {
+  errors: TypecheckError[];
+  total: number;
+  suppressed: number;
+} {
+  const errors: TypecheckError[] = [];
+  let total = 0;
+  let suppressed = 0;
+  let last: TypecheckError | null = null;
+  for (const line of output.split("\n")) {
+    const m = /^(.+?)\((\d+),(\d+)\): error (TS\d+): (.*)$/.exec(line);
+    if (!m) {
+      // Elaboration lines are indented; fold them into the last kept error.
+      if (last && /^\s+\S/.test(line)) last.message += `\n${line.trimEnd()}`;
+      continue;
+    }
+    const [, rawFile, ln, col, code, message] = m;
+    // Keep only diagnostics in the user's files. tsc runs with cwd=APP_DIR
+    // (the daemon's cwd), so project files come out relative ("tests/x.ts")
+    // and anything outside — the SDK at /opt/spectest/sdk, node_modules —
+    // is `../…` or absolute. Errors in our own SDK are ours to fix, not the
+    // user's to read.
+    const abs = path.resolve(APP_DIR, rawFile!);
+    const inApp = abs.startsWith(APP_DIR + path.sep) && !abs.includes(`${path.sep}node_modules${path.sep}`);
+    if (!inApp) {
+      suppressed += 1;
+      last = null;
+      continue;
+    }
+    total += 1;
+    const err: TypecheckError = {
+      file: path.relative(APP_DIR, abs),
+      line: Number(ln),
+      column: Number(col),
+      code: code!,
+      message: message!,
+    };
+    if (errors.length < TYPECHECK_ERROR_CAP) {
+      errors.push(err);
+      last = err;
+    } else {
+      last = null;
+    }
+  }
+  return { errors, total, suppressed };
+}
+
+async function runTypecheck(): Promise<TypecheckReport> {
+  const started = Date.now();
+  // The project's own typescript wins over the baked copy.
+  const tsc = [
+    path.join(APP_DIR, "node_modules", "typescript", "bin", "tsc"),
+    path.join(TYPECHECK_DIR, "node_modules", "typescript", "bin", "tsc"),
+  ].find((p) => existsSync(p));
+  if (!tsc) {
+    return {
+      status: "skipped",
+      errors: [],
+      totalErrors: 0,
+      durationMs: 0,
+      detail: "typescript is not installed (base image predates the baked typechecker)",
+    };
+  }
+  let config = path.join(APP_DIR, "tsconfig.json");
+  if (!existsSync(config)) {
+    config = path.join(APP_DIR, ".spectest-tsconfig.json");
+    await fs.writeFile(config, generatedTsconfig());
+  }
+  const res = await shx(process.execPath, [tsc, "--noEmit", "--pretty", "false", "-p", config], 120_000);
+  const durationMs = Date.now() - started;
+  if (res.code === 124) {
+    return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: "typecheck timed out" };
+  }
+  const { errors, total, suppressed } = parseTscOutput(res.stdout + res.stderr);
+  if (errors.length === 0) {
+    // Exit 0 → clean. Non-zero with no *user-file* diagnostics is either
+    // all-suppressed (still ok from the user's perspective) or a compiler
+    // crash (config not found, OOM) — surface the latter.
+    if (res.code !== 0 && suppressed === 0) {
+      const tail = (res.stderr || res.stdout).trim().split("\n").slice(-5).join("\n");
+      return { status: "failed", errors: [], totalErrors: 0, durationMs, detail: tail || `tsc exited ${res.code}` };
+    }
+    if (suppressed > 0) {
+      console.warn(`[typecheck] ${suppressed} diagnostic(s) outside the project suppressed`);
+    }
+    return { status: "ok", errors: [], totalErrors: 0, durationMs };
+  }
+  return { status: "errors", errors, totalErrors: total, durationMs };
+}
+
 // ────────────────────────────────────────────────────────────────────────
 // HTTP server
 // ────────────────────────────────────────────────────────────────────────
@@ -4464,6 +4707,11 @@ async function handle(req: http.IncomingMessage, res: http.ServerResponse, state
     // resulting catalogue. Called after the warm snapshot (cold path) or
     // against a freshly restored VM (warm path).
     await loadTests();
+    // Background typecheck of the freshly-uploaded app tree — both paths
+    // (cold and warm) funnel through here after the upload, so the check
+    // always sees the current code. Never on the critical path: the reply
+    // doesn't wait, POST /typecheck collects.
+    startTypecheck();
     const l = requireLoaded();
     jsonResponse(res, 200, {
       environment: l.project.environment,
@@ -4473,6 +4721,15 @@ async function handle(req: http.IncomingMessage, res: http.ServerResponse, state
     return;
   }
 
+  if (method === "POST" && url === "/typecheck") {
+    // Await the run kicked off at /load-tests (or start one on demand —
+    // e.g. a legacy single-file project loaded before this daemon shipped
+    // the kickoff, or a direct debug call).
+    if (!TYPECHECK) startTypecheck();
+    jsonResponse(res, 200, await TYPECHECK!);
+    return;
+  }
+
   if (method === "POST" && url === "/unload") {
     loaded = null;
     jsonResponse(res, 200, { unloaded: true });
@@ -4626,13 +4883,55 @@ async function handle(req: http.IncomingMessage, res: http.ServerResponse, state
     state.inFlightTest = exec;
     try {
       const result = await exec;
-      jsonResponse(res, 200, result);
+      // Recordings go out-of-band: encode + park the bundle, reply with a
+      // size ref only (see the REPLAY_BUNDLES comment). A bundle-encoding
+      // failure drops the recordings (warn) rather than failing the case —
+      // and never falls back to inlining, which is exactly the >16 MB
+      // response this path exists to avoid.
+      const { browserSessions, ...wire } = result;
+      let replay: { bytes: number } | undefined;
+      if (browserSessions.length > 0) {
+        try {
+          const gz = encodeReplayBundle(caseId, browserSessions);
+          stashReplayBundle(caseId, gz);
+          replay = { bytes: gz.length };
+        } catch (err) {
+          // eslint-disable-next-line no-console
+          console.warn("[replay] bundle encode failed; dropping recordings:", err);
+        }
+      }
+      jsonResponse(res, 200, { ...wire, replay });
     } finally {
       state.inFlightTest = null;
     }
     return;
   }
 
+  if (method === "POST" && url === "/replay-chunk") {
+    // One chunk of a parked replay bundle (see REPLAY_BUNDLES). POST with
+    // a JSON body — case ids are arbitrary user strings, and a body dodges
+    // URL-encoding across both providers' daemon_http transports.
+    const body = await readBody(req);
+    let parsed: { caseId?: string; offset?: number; limit?: number };
+    try {
+      parsed = JSON.parse(body || "{}");
+    } catch {
+      jsonResponse(res, 400, { error: "invalid JSON body" });
+      return;
+    }
+    if (!parsed.caseId) {
+      jsonResponse(res, 400, { error: "caseId is required" });
+      return;
+    }
+    const gz = REPLAY_BUNDLES.get(parsed.caseId);
+    if (!gz) {
+      jsonResponse(res, 404, { error: `no replay bundle for case: ${parsed.caseId}` });
+      return;
+    }
+    jsonResponse(res, 200, replayChunk(gz, parsed.offset, parsed.limit));
+    return;
+  }
+
   jsonResponse(res, 404, { error: "not found" });
 }