@specific.dev/spectest 0.12.0 → 0.14.0

package/src/daemon.ts

@@ -65,6 +65,8 @@ import type {
   BrowserSessionStep,
 } from "./browser.js";
 import type {
+  ComponentContext,
+  ComponentExecOpts,
   EnvironmentConfig,
   ExecResult,
   FakeContext,
@@ -607,6 +609,16 @@ function sanitizeSegment(p: string): string {
 }
 
 function resolveHostPath(service: string, vol: VolumeMount): string {
+  if (vol.name) {
+    // Named shared volume: one backing dir per name, shared by every
+    // service that mounts the same name (storage-api ↔ imgproxy). Rooted
+    // in the per-env state tree (or the cache tree when cache-flagged),
+    // so teardown/fork semantics match ordinary volumes.
+    const root = vol.cache
+      ? ["/var/cache/spectest/volumes", "_shared"]
+      : [WORKSPACE, ".spectest", "volumes", "_shared"];
+    return path.join(...root, sanitizeSegment(vol.name));
+  }
   if (vol.source && vol.source.startsWith("/")) return vol.source;
   // Cache volumes root OUTSIDE /workspace so the delta-restore teardown
   // (rm -rf /workspace) keeps them — they hold only content-addressed
@@ -640,6 +652,13 @@ async function ensureVolumes(svc: NamedService): Promise<string[]> {
   const flags: string[] = [];
   if (!svc.volumes || svc.volumes.length === 0) return flags;
   for (const vol of svc.volumes) {
+    // Boot services are validated in defineEnvironment; re-check here so
+    // runtime `startService` specs get the same contract.
+    if (vol.name && vol.source) {
+      throw new Error(
+        `service "${svc.name}" volume for ${JSON.stringify(vol.target)} sets both \`name\` and \`source\``,
+      );
+    }
     const host = resolveHostPath(svc.name, vol);
     await fs.mkdir(host, { recursive: true });
     if (vol.source?.startsWith("/") && !host.startsWith("/var/cache/spectest/")) {
@@ -1063,12 +1082,21 @@ async function probeTcp(host: string, port: number): Promise<boolean> {
   });
 }
 
-async function probeHttp(host: string, port: number, urlPath: string): Promise<boolean> {
+async function probeHttp(
+  host: string,
+  port: number,
+  urlPath: string,
+  headers?: Record<string, string>,
+  expectStatus?: number,
+): Promise<boolean> {
   const ctrl = new AbortController();
   const to = setTimeout(() => ctrl.abort(), 5000);
   try {
-    const res = await fetch(`http://${host}:${port}${urlPath}`, { signal: ctrl.signal });
-    return res.ok;
+    const res = await fetch(`http://${host}:${port}${urlPath}`, {
+      signal: ctrl.signal,
+      headers,
+    });
+    return expectStatus !== undefined ? res.status === expectStatus : res.ok;
   } catch {
     return false;
   } finally {
@@ -1098,7 +1126,13 @@ async function waitForReady(svc: NamedService): Promise<void> {
     if (check.type === "tcp") {
       ok = await probeTcp(svc.name, check.port);
     } else if (check.type === "http") {
-      ok = await probeHttp(svc.name, check.port, check.path ?? "/");
+      ok = await probeHttp(
+        svc.name,
+        check.port,
+        check.path ?? "/",
+        check.headers,
+        check.expectStatus,
+      );
     } else {
       ok = await probeExec(svc.name, check.command);
     }
@@ -2510,7 +2544,7 @@ async function bootstrap(): Promise<BootstrapTimings> {
     if (svc.setup) {
       progressService(svc.name, { status: "probing", detail: "running setup" });
       const helpers = await ensureHelpers(svc.name, svc);
-      await svc.setup({ name: svc.name, helpers });
+      await svc.setup({ name: svc.name, helpers, ...componentContext() });
     }
     progressService(svc.name, { status: "ready", detail: undefined });
     const ti = timings.get(svc.name);
@@ -2618,71 +2652,168 @@ interface RunResult {
   /** asciicast sessions for each `ctx.terminal(...)` call. */
   terminalSessions: TerminalSessionRecord[];
   /**
-   * `docker logs` per service, captured only when the test failed (empty
-   * on a pass). Lets the failure post-mortem in the web UI show what each
-   * container printed without the author having to add `ctx.exec` log
-   * grabs by hand.
+   * Per-service log *delta* — the lines each service emitted during THIS
+   * case, beyond what ancestor cases already captured. Captured on every
+   * case (pass or fail). Because a child fork restores the parent's log
+   * markers (module memory travels with the snapshot), these deltas tile
+   * along the ancestor chain into the full cumulative log for any branch —
+   * reconstructed on the dashboard; the CLI failure post-mortem shows the
+   * failing case's own delta. See {captureServiceLogDeltas}.
    */
-  serviceLogs: ServiceLogCapture[];
+  serviceLogDeltas: ServiceLogDelta[];
   error?: { message: string; stack?: string };
 }
 
-/** Captured container logs for one service. */
-interface ServiceLogCapture {
+// ────────────────────────────────────────────────────────────────────────
+// Cumulative service logs (per-case deltas → S3, reconstructed on the web)
+// ────────────────────────────────────────────────────────────────────────
+//
+// A child test runs in a fork restored from its parent's memory+filesystem
+// snapshot, so `docker logs <svc>` in the child already contains the
+// parent's entire history plus the child's own output — logs are
+// inherently cumulative along each branch. Rather than store the (growing,
+// redundant) full log at every case, each case stores only its DELTA (the
+// lines it added), and the dashboard reconstructs a branch's full log by
+// concatenating deltas along the ancestor chain.
+//
+// The load-bearing trick: `LOG_MARKERS` lives in module memory, so it
+// FORKS with the snapshot (same mechanism as `TEST_DATA` / `RUNTIME_SERVICES`).
+// `runOne` advances the markers BEFORE `/run` returns, and the control
+// plane snapshots the fork AFTER `/run` returns — so a child (forked or
+// handed off) restores the parent's final markers and its delta tiles on
+// with no gap or cross-branch duplication.
+
+/**
+ * Per-(service, stream) count of newline-terminated log lines already
+ * captured by an ancestor case. Key is `"<service><stream>"`. The
+ * next case on this branch captures only `lines[marker..]`. Module-scope
+ * so it travels with the snapshot into every fork.
+ */
+const LOG_MARKERS = new Map<string, { lines: number }>();
+
+/** Per-(service, stream) delta byte cap. Over this we keep head+tail and
+ *  elide the middle — the head preserves the continuation from the parent,
+ *  the tail preserves the newest output — while still advancing the marker
+ *  to the true line count so the chain stays aligned. */
+const LOG_DELTA_MAX_BYTES = 2 * 1024 * 1024;
+
+/** One service's per-case log delta (new output since the parent case). */
+interface ServiceLogDelta {
   service: string;
-  /** Trailing slice of the container's stdout (RFC3339-timestamped). */
   stdout: string;
   stdoutTruncated: boolean;
-  /** Trailing slice of the container's stderr. */
+  /** The container's stdout shrank below our marker (it was recreated /
+   *  rotated), so this delta is the full current log, not a continuation. */
+  stdoutReset: boolean;
   stderr: string;
   stderrTruncated: boolean;
+  stderrReset: boolean;
 }
 
-/** How many trailing log lines to grab per service on failure. */
-const SERVICE_LOG_TAIL_LINES = 500;
-/** Per-stream byte cap after the line tail (keeps the most recent bytes). */
-const SERVICE_LOG_MAX_BYTES = 256 * 1024;
-
-/** Keep the trailing `max` bytes of `s` — the opposite of
- *  `truncateUtf8`'s head-keep, because the most recent output is what
- *  explains a failure. */
-function tailBytes(s: string, max: number): { value: string; truncated: boolean } {
+/** Keep the head and tail of `s`, eliding the middle when it exceeds
+ *  `max` (string length, a byte proxy as elsewhere here). Head+tail so an
+ *  over-long delta keeps both the parent-continuation and the newest
+ *  output. */
+function capMiddle(s: string, max: number): { value: string; truncated: boolean } {
   if (s.length <= max) return { value: s, truncated: false };
-  return { value: s.slice(s.length - max), truncated: true };
+  const half = Math.floor(max / 2);
+  const elided = s.length - 2 * half;
+  return {
+    value: `${s.slice(0, half)}\n… [${elided} bytes elided] …\n${s.slice(s.length - half)}`,
+    truncated: true,
+  };
+}
+
+/**
+ * Compute one stream's delta beyond `marker` complete lines.
+ *  - Counts only newline-terminated lines; a trailing partial line (no
+ *    `\n` yet) is held back from both the delta and the count, so a line
+ *    completed by a later capture isn't split across the fork boundary.
+ *  - Reset guard: if the stream shrank below `marker` (container recreated
+ *    or rotated) the whole current log is re-emitted and `reset` is set.
+ */
+function streamDelta(
+  full: string,
+  marker: number,
+): { delta: string; total: number; reset: boolean; truncated: boolean } {
+  const lastNl = full.lastIndexOf("\n");
+  const complete = lastNl < 0 ? "" : full.slice(0, lastNl + 1);
+  let total = 0;
+  for (let i = 0; i < complete.length; i++) {
+    if (complete.charCodeAt(i) === 10) total++;
+  }
+  let reset = false;
+  let startLine = marker;
+  if (total < marker) {
+    reset = true;
+    startLine = 0;
+  }
+  let delta: string;
+  if (startLine <= 0) {
+    delta = complete;
+  } else if (startLine >= total) {
+    delta = "";
+  } else {
+    // Byte offset just past the `startLine`-th newline.
+    let seen = 0;
+    let off = 0;
+    for (let i = 0; i < complete.length; i++) {
+      if (complete.charCodeAt(i) === 10 && ++seen === startLine) {
+        off = i + 1;
+        break;
+      }
+    }
+    delta = complete.slice(off);
+  }
+  const capped = capMiddle(delta, LOG_DELTA_MAX_BYTES);
+  return { delta: capped.value, total, reset, truncated: capped.truncated };
 }
 
 /**
- * Capture `docker logs` for every service in the loaded environment.
- * Called when a test case fails so the web UI can show each container's
- * recent output. Best-effort and bounded: the last
- * {SERVICE_LOG_TAIL_LINES} lines, trimmed to the trailing
- * {SERVICE_LOG_MAX_BYTES} bytes per stream. A `docker logs` failure for
- * one service surfaces as that service's `stderr` rather than aborting
- * the whole capture, so a crashed/removed container is still visible.
+ * Capture the per-service log delta for the current case and advance the
+ * markers. Runs on EVERY case (pass or fail). Enumerates boot services
+ * (`namedServices`) plus any runtime services this fork started
+ * (`RUNTIME_SERVICES`), deduped by name. A `docker logs` failure surfaces
+ * as the service's `stderr` WITHOUT advancing the markers — a transient
+ * failure must never desync the chain.
  */
-async function captureServiceLogs(): Promise<ServiceLogCapture[]> {
+async function captureServiceLogDeltas(): Promise<ServiceLogDelta[]> {
   const l = loaded;
   if (!l) return [];
-  // Boot services plus any runtime services this fork started — both are
-  // real containers a failure post-mortem wants to see. Dedup by name.
   const byName = new Map<string, NamedService>();
   for (const s of namedServices(l.project.environment)) byName.set(s.name, s);
   for (const [name, s] of RUNTIME_SERVICES) byName.set(name, s);
   const services = [...byName.values()];
   return Promise.all(
-    services.map(async (svc): Promise<ServiceLogCapture> => {
-      const r = await docker(
-        ["logs", "--tail", String(SERVICE_LOG_TAIL_LINES), "--timestamps", svc.name],
-        30_000,
-      );
-      const stdout = tailBytes(r.stdout, SERVICE_LOG_MAX_BYTES);
-      const stderr = tailBytes(r.stderr, SERVICE_LOG_MAX_BYTES);
+    services.map(async (svc): Promise<ServiceLogDelta> => {
+      const r = await docker(["logs", "--timestamps", svc.name], 30_000);
+      if (r.code !== 0) {
+        // Container gone/renamed — surface the CLI error, leave markers put.
+        const err = capMiddle(r.stderr || r.stdout, LOG_DELTA_MAX_BYTES);
+        return {
+          service: svc.name,
+          stdout: "",
+          stdoutTruncated: false,
+          stdoutReset: false,
+          stderr: err.value,
+          stderrTruncated: err.truncated,
+          stderrReset: false,
+        };
+      }
+      const outKey = `${svc.name}stdout`;
+      const errKey = `${svc.name}stderr`;
+      const out = streamDelta(r.stdout, LOG_MARKERS.get(outKey)?.lines ?? 0);
+      const err = streamDelta(r.stderr, LOG_MARKERS.get(errKey)?.lines ?? 0);
+      LOG_MARKERS.set(outKey, { lines: out.total });
+      LOG_MARKERS.set(errKey, { lines: err.total });
       return {
         service: svc.name,
-        stdout: stdout.value,
-        stdoutTruncated: stdout.truncated,
-        stderr: stderr.value,
-        stderrTruncated: stderr.truncated,
+        stdout: out.delta,
+        stdoutTruncated: out.truncated,
+        stdoutReset: out.reset,
+        stderr: err.delta,
+        stderrTruncated: err.truncated,
+        stderrReset: err.reset,
       };
     }),
   );
@@ -2881,6 +3012,64 @@ async function openInstrumentedTerminal(
 // already populated. Carries arbitrary JS values — no JSON round-trip.
 const TEST_DATA = new Map<string, unknown>();
 
+// ────────────────────────────────────────────────────────────────────────
+// Component context — the exec / project-file surface handed to service
+// `setup` hooks and `helpers` factories (`ComponentContext` in index.ts),
+// so components don't hand-roll child_process docker execs or hard-code
+// control-plane paths like /workspace.
+// ────────────────────────────────────────────────────────────────────────
+
+const COMPONENT_EXEC_DEFAULT_TIMEOUT_MS = 120_000;
+
+/** Raw `docker exec` with optional piped stdin. Array command = exact
+ * argv (no shell); string = `sh -lc`. Non-zero exit is reported via
+ * `exitCode`, never thrown. Unlike `execInService` this records nothing —
+ * setup/helpers-factory time has no test timeline. */
+function componentExec(
+  service: string,
+  command: string | string[],
+  opts?: ComponentExecOpts,
+): Promise<ExecResult> {
+  const argv = ["exec", "-i"];
+  if (opts?.cwd) argv.push("-w", opts.cwd);
+  argv.push(service);
+  if (typeof command === "string") argv.push("sh", "-lc", command);
+  else argv.push(...command);
+  const timeoutMs = opts?.timeoutMs ?? COMPONENT_EXEC_DEFAULT_TIMEOUT_MS;
+  return new Promise((resolve, reject) => {
+    const child = spawn("docker", argv, {
+      stdio: [opts?.stdin !== undefined ? "pipe" : "ignore", "pipe", "pipe"],
+    });
+    const out: Buffer[] = [];
+    const err: Buffer[] = [];
+    child.stdout!.on("data", (c: Buffer) => out.push(c));
+    child.stderr!.on("data", (c: Buffer) => err.push(c));
+    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+    child.on("error", (e) => {
+      clearTimeout(timer);
+      reject(e);
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({
+        stdout: Buffer.concat(out).toString("utf8"),
+        stderr: Buffer.concat(err).toString("utf8"),
+        exitCode: code ?? -1,
+      });
+    });
+    if (opts?.stdin !== undefined) child.stdin!.end(opts.stdin);
+  });
+}
+
+function componentContext(): ComponentContext {
+  return {
+    projectRoot: WORKSPACE,
+    readProjectFile: (p: string) =>
+      fs.readFile(path.isAbsolute(p) ? p : path.join(WORKSPACE, p), "utf8"),
+    exec: componentExec,
+  };
+}
+
 // Cached helper namespaces produced by `ServiceDefinition.helpers`
 // factories. Built lazily on first access and reused for the daemon's
 // lifetime — Bun.SQL pools and similar resources are happy to live a
@@ -2900,7 +3089,7 @@ async function ensureHelpers(
 ): Promise<Record<string, unknown>> {
   if (!def.helpers) return {};
   if (!HELPERS_CACHE.has(name)) {
-    HELPERS_CACHE.set(name, await def.helpers({ name }));
+    HELPERS_CACHE.set(name, await def.helpers({ name, ...componentContext() }));
   }
   return HELPERS_CACHE.get(name)!;
 }
@@ -3460,7 +3649,8 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
   // `finally` — leaked Chromium subprocesses would survive the snapshot
   // and chew memory across forks. Each Browser also gets a session
   // recorder; the records flow back to the control plane as part of
-  // RunResult.browserSessions and are persisted to SQLite.
+  // RunResult.browserSessions and are archived to S3 as the case's
+  // replay bundle.
   // Tracks both Browser and Mobile handles for cleanup — both expose an
   // async close() that does the final rrweb drain before teardown.
   const openBrowsers: Array<{ close(): Promise<void> }> = [];
@@ -3564,17 +3754,20 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     for (const s of sessions) s.markClosed();
   }
   const durationMs = Date.now() - start;
-  // On failure, grab each service's recent container logs for the
-  // post-mortem. Captured after the duration clock stops so the
-  // log-fetch round trips aren't billed to the test.
-  let serviceLogs: ServiceLogCapture[] = [];
-  if (outcome.status === "failed") {
-    try {
-      serviceLogs = await captureServiceLogs();
-    } catch (err) {
-      // eslint-disable-next-line no-console
-      console.warn("[service-logs] capture failed:", err);
-    }
+  // Capture each service's log delta (the lines THIS case added beyond its
+  // ancestors) on every case, pass or fail. Runs after the duration clock
+  // stops so the `docker logs` round trips aren't billed to the test.
+  // Advancing the markers here — before /run returns and the control plane
+  // snapshots the fork — is what lets children tile their own deltas on
+  // seamlessly. Shipped to S3: the dashboard reconstructs the full
+  // cumulative log for a branch, the CLI failure post-mortem shows the
+  // failing case's own delta.
+  let serviceLogDeltas: ServiceLogDelta[] = [];
+  try {
+    serviceLogDeltas = await captureServiceLogDeltas();
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.warn("[service-logs] delta capture failed:", err);
   }
   const events = stopRecording();
   // Drop sessions whose linking event didn't survive — an exec/terminal
@@ -3593,7 +3786,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     events,
     browserSessions: sessions.map((s) => s.record),
     terminalSessions: terminalSessions.filter((s) => referencedSessions.has(s.sessionId)),
-    serviceLogs,
+    serviceLogDeltas,
     error: outcome.error,
   };
 }
@@ -4080,6 +4273,24 @@ async function handle(req: http.IncomingMessage, res: http.ServerResponse, state
     return;
   }
 
+  if (method === "POST" && url === "/capture-log-baseline") {
+    // Snapshot each service's log output produced during env bring-up
+    // (container startup + project setup), advancing the log markers so the
+    // subsequent per-test deltas start AFTER setup. The control plane calls
+    // this once, on the main env, before the first test runs — see
+    // tests.rs::capture_log_baseline. Best-effort: on failure the setup lines
+    // simply fold into the first test's delta as before.
+    let serviceLogDeltas: ServiceLogDelta[] = [];
+    try {
+      serviceLogDeltas = await captureServiceLogDeltas();
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.warn("[service-logs] baseline capture failed:", err);
+    }
+    jsonResponse(res, 200, { serviceLogDeltas });
+    return;
+  }
+
   if (method === "POST" && url === "/run") {
     if (state.inFlightTest) {
       jsonResponse(res, 409, { error: "another test is already running" });