@specific.dev/spectest 0.57.0 → 0.59.0

package/dist/harness/coverage.js

@@ -2,12 +2,15 @@
 //
 // A service that opts in (`coverage: true | { command }`) gets a writable
 // directory bind-mounted at `/spectest/coverage/` in its container. The
-// service's own tool writes lcov reports there. The daemon reads the
+// service's own tool, or its coverage adapters, write reports there —
+// lcov, or the V8 coverage JSON that `NODE_V8_COVERAGE` writes. The daemon reads the
 // directory from the VM side (no `docker exec` to fetch) after bring-up
 // and after each test, and ships the reports **verbatim** to the control
 // plane as a gzipped bundle. Nothing is interpreted here beyond the one
-// contract the harness must fail a case on: at least one report, and at
-// least one lcov `SF:` record among them. The server derives whatever it
+// contract the harness must fail a case on: at least one report, every
+// one well-formed. Counters may reset between dumps and a dump may be a
+// new file: the directory forks with the environment, so it accumulates
+// along the ancestor chain by itself. The server derives whatever it
 // needs (today the set of source files; later, finer things) from the
 // stored bytes, so a change in that derivation is a re-read of stored
 // runs, never a hole in the history.
@@ -19,7 +22,7 @@ import { gzipSync } from "node:zlib";
 /** Where the directory is mounted inside an opted-in container. */
 export const COVERAGE_CONTAINER_DIR = "/spectest/coverage";
 /** Raw bytes of reports one service may ship per capture. Reports are
- *  cumulative and bounded by code size, so anything past this is a
+ *  bounded by code size times chain depth, so anything past this is a
  *  runaway (a tool writing a new file per request), not coverage. */
 export const COVERAGE_MAX_BYTES_PER_SERVICE = 64 * 1024 * 1024;
 /** Root of the per-service host directories, under the workspace so the
@@ -31,6 +34,34 @@ export function coverageHostDir(workspace, service) {
 export function hasLcovSourceFile(text) {
     return /^SF:\S/m.test(text);
 }
+/** True when `text` is lcov-shaped: at least one line, every non-blank
+ *  line an `XX:` record or `end_of_record`. An lcov with no `SF:` record
+ *  at all (`TN:` alone) is well-formed — it is a report that says
+ *  nothing ran. */
+export function isLcovShaped(text) {
+    const lines = text.split(/\r?\n/).filter((l) => l.trim().length > 0);
+    return lines.length > 0 && lines.every((l) => /^[A-Z]{2,5}:/.test(l) || l.trim() === "end_of_record");
+}
+/**
+ * True when `text` looks like a V8 coverage JSON document — what
+ * `NODE_V8_COVERAGE` / `v8.takeCoverage()` write: an object with a
+ * `result` array of scripts. A cheap shape check, not a parse (a report
+ * can be megabytes); the control plane parses for real.
+ */
+export function isV8CoverageJson(text) {
+    return /^\s*\{/.test(text) && /"result"\s*:\s*\[/.test(text);
+}
+/** True when `text` is a report spectest stores: lcov or V8 JSON. */
+export function isWellFormedReport(text) {
+    return isLcovShaped(text) || isV8CoverageJson(text);
+}
+/** True when a well-formed report names no source file at all (an lcov
+ *  with no `SF:`, a V8 document with an empty `result`). */
+export function isEmptyReport(text) {
+    if (isV8CoverageJson(text))
+        return /"result"\s*:\s*\[\s*\]/.test(text);
+    return !hasLcovSourceFile(text);
+}
 /** Every regular file under `dir`, recursively, as paths relative to it.
  *  Hidden files are skipped (a tool's own bookkeeping — `.seq`, `.lock`,
  *  a tmp file mid-write — is not a report). */
@@ -58,9 +89,11 @@ export async function listReportFiles(dir) {
     return out.sort();
 }
 /**
- * Read every report under `dir`, verbatim. An empty directory, or one with
- * no `SF:` record anywhere, is an error rather than an empty capture: the
- * contract is that an opted-in service always has a report to read.
+ * Read every report under `dir`, verbatim. An empty directory, or a file
+ * that is neither lcov nor V8 JSON, is an error rather than an empty
+ * capture: the contract is that an opted-in service always has a
+ * well-formed report to read. A report that names no file is fine — it
+ * says nothing ran.
  */
 export async function readCoverageDir(service, dir) {
     const names = await listReportFiles(dir);
@@ -68,12 +101,13 @@ export async function readCoverageDir(service, dir) {
         return {
             service,
             reports: [],
-            error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov file there`,
+            error: `service "${service}" opted in to coverage but ${COVERAGE_CONTAINER_DIR}/ holds no report — the service (or its coverage command) must write an lcov or V8 coverage JSON file there`,
         };
     }
     const reports = [];
     let total = 0;
-    let anyLcov = false;
+    let anyReport = false;
+    const malformed = [];
     for (const name of names) {
         let content;
         try {
@@ -87,18 +121,20 @@ export async function readCoverageDir(service, dir) {
             return {
                 service,
                 reports: [],
-                error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — reports must be cumulative, not one file per dump`,
+                error: `service "${service}" has more than ${COVERAGE_MAX_BYTES_PER_SERVICE >> 20} MiB of reports in ${COVERAGE_CONTAINER_DIR}/ — that is a runaway, not coverage`,
             };
         }
-        if (hasLcovSourceFile(content))
-            anyLcov = true;
+        if (isWellFormedReport(content))
+            anyReport = true;
+        else
+            malformed.push(name);
         reports.push({ name, content });
     }
-    if (!anyLcov) {
+    if (!anyReport || malformed.length > 0) {
         return {
             service,
             reports,
-            error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but none holds an lcov \`SF:\` record — only lcov reports are read`,
+            error: `service "${service}" wrote ${reports.length} file(s) to ${COVERAGE_CONTAINER_DIR}/ but ${anyReport ? `${malformed.slice(0, 5).join(", ")} ${malformed.length === 1 ? "is" : "are"} neither` : "none is"} an lcov report or a V8 coverage JSON document — only those two formats are read`,
         };
     }
     return { service, reports };

package/dist/index.d.ts

@@ -20,6 +20,8 @@ export type { Mobile, MobileApp } from "./mobile.js";
 import type { Mobile, MobileApp } from "./mobile.js";
 export type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
 import type { Terminal, TerminalOpts, TerminalResult } from "./terminal.js";
+export { type CoverageAdapter, type CoverageCaptureContext, type CoverageConfigureInfo, type CoverageLoadContext, type ServiceCoverage, } from "./coverage.js";
+import { type ServiceCoverage } from "./coverage.js";
 export { certificate, dnsName, proxy, provides, lowerIngress, isWildcard, SELF_SERVICE_TOKEN, } from "./ingress.js";
 export type { CertificateDecl, DnsDecl, ProxyDecl, IngressDecl, DnsTarget, LoweredIngress, } from "./ingress.js";
 import type { DnsTarget } from "./ingress.js";
@@ -119,20 +121,24 @@ export interface ServiceConfig {
      * Opt this service in to **code coverage collection** (experimental).
      *
      * The service gets a writable directory bind-mounted at
-     * `/spectest/coverage/`. The service's own coverage tool writes **lcov**
-     * reports there — cumulative from process start, never reset. After
-     * bring-up and after every test, spectest runs `command` (if given) inside
-     * the container, then reads every file in the directory and records the
-     * set of source files that ran. Only the `SF:` records are read.
+     * `/spectest/coverage/`. After bring-up and after every test, spectest
+     * reads every file in it — **lcov** or **V8 coverage JSON**, verbatim —
+     * and stores them with the test; the control plane derives what ran.
      *
-     * - `true`: the service writes its reports by itself (a dump on a timer,
-     *   an exit-time dump for short-lived processes).
-     * - `{ command }`: run this (via `sh -c`) in the container to make the
-     *   service write a report, before the directory is read.
+     * - `true`: the program writes its own reports there (an exit-time dump
+     *   for a short-lived process, a dump on a timer, its own hook).
+     * - `{ adapters: [...] }`: spectest gets the reports out. From
+     *   `@specific.dev/spectest/coverage`: `node()` (V8 JSON from a node
+     *   service, through a hook spectest mounts), `browser()` (the frontend
+     *   this service serves, collected from the guest browser),
+     *   `command(cmd)` (run a command in the container first), or your own
+     *   via `defineAdapter`.
      *
-     * The directory must hold at least one lcov report each time it is read;
-     * an empty directory or a failed command fails the test with a clear
-     * error, so a hole in the coverage data is never silent. See
+     * The directory must hold at least one well-formed report each time it
+     * is read; an empty directory or a failing adapter fails the test with a
+     * clear error, so a hole in the coverage data is never silent. Reports
+     * accumulate along the test's ancestor chain by themselves (the
+     * directory forks with the environment). See
      * `spectest docs /services/coverage`.
      */
     coverage?: ServiceCoverage;
@@ -663,14 +669,6 @@ export type ServiceImage = {
     /** Extra glob patterns to exclude from the build context. */
     exclude?: readonly string[];
 };
-/**
- * A service's opt-in to coverage collection — see `ServiceConfig.coverage`.
- * `true` when the service writes its own reports; `{ command }` when
- * spectest must run a command in the container to produce one.
- */
-export type ServiceCoverage = true | {
-    command: string;
-};
 export interface VolumeMount {
     /**
      * Named shared volume. Two services mounting the same `name` share one
@@ -883,11 +881,6 @@ export interface FakeContext {
      *  for a TLS-provisioning provider hands back to the app under test. */
     certificate(hostnames: readonly string[]): Promise<CertificateMaterial>;
 }
-/**
- * Check one service's `coverage` field. Exported so the daemon applies the
- * same rule to a runtime `startService` spec.
- */
-export declare function validateCoverage(service: string, cov: unknown): void;
 /**
  * A single test step. Created via `test(...)` or `createTest(services)`.
  * Tests are referenced (not named by string) when one test depends on

package/dist/index.js

@@ -40,6 +40,7 @@ export { McpHttpError, McpRpcError, McpAuthDeniedError, } from "./mcp.js";
 import { isLocator, getLocatorProbe, isBrowserSession, getBrowserProbe, DEFAULT_ACTION_TIMEOUT_MS, } from "./locator.js";
 import { formatWaited, locatorFailureMessage } from "./locator-errors.js";
 import { describeUrlPattern, matchesUrl } from "./url-match.js";
+import { applyCoverageAdapters, validateCoverage } from "./coverage.js";
 // Low-level ingress primitives + the framework lowering that the friendly
 // `tls` / `hostnames` fields and `defineFake(...)` are built on. See
 // `ingress.ts`.
@@ -215,21 +216,6 @@ function expandServiceGroups(input) {
     }
     return out;
 }
-/**
- * Check one service's `coverage` field. Exported so the daemon applies the
- * same rule to a runtime `startService` spec.
- */
-export function validateCoverage(service, cov) {
-    if (cov === undefined || cov === true)
-        return;
-    if (typeof cov === "object" &&
-        cov !== null &&
-        typeof cov.command === "string" &&
-        cov.command.trim().length > 0) {
-        return;
-    }
-    throw new Error(`service "${service}" has an invalid \`coverage\` value — use \`true\` or \`{ command: "<shell command>" }\``);
-}
 function validateEnvironmentConfig(config) {
     const entries = Object.entries(config.services);
     const serviceNames = new Set(entries.map(([n]) => n));
@@ -337,9 +323,13 @@ export function defineEnvironment(input) {
     // { name, services, timeoutSecs } — with plain, fully-expanded services —
     // is stored as `config` and shipped to the control plane.
     const { fakes, services: rawServices, ...rest } = input;
+    const expanded = expandServiceGroups(rawServices);
+    for (const [key, svc] of Object.entries(expanded)) {
+        expanded[key] = applyCoverageAdapters(key, svc);
+    }
     const config = {
         ...rest,
-        services: expandServiceGroups(rawServices),
+        services: expanded,
     };
     validateEnvironmentConfig(config);
     if (fakes)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.57.0",
+  "version": "0.59.0",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",
@@ -16,6 +16,11 @@
       "bun": "./src/components/index.ts",
       "default": "./dist/components/index.js"
     },
+    "./coverage": {
+      "types": "./dist/coverage.d.ts",
+      "bun": "./src/coverage.ts",
+      "default": "./dist/coverage.js"
+    },
     "./browser": {
       "types": "./dist/browser.d.ts",
       "bun": "./src/browser.ts",

package/src/browser-coverage.ts

@@ -0,0 +1,209 @@
+// Browser coverage — the collector.
+//
+// One collector per CDP session (i.e. per page view). `attach` turns on
+// V8 precise coverage and tracks parsed scripts; `harvest` takes the
+// counters (which RESETS them — V8 reports nothing for a script that ran
+// nothing since the last take, so the counts are summed here across
+// takes), maps each script the browser loaded from a covered service back
+// to its original sources through the served source map, and folds the
+// result into a per-service total in module memory — which forks with the
+// daemon, so a child test inherits its ancestors' browser coverage exactly
+// as a container's counters are inherited.
+//
+// Harvest runs before every browser step (a click that navigates
+// cross-site replaces the renderer, and the old counters with it) and at
+// every coverage capture. `browserCoverageReports` renders the totals as
+// one lcov per service for the `case-coverage` bundle.
+//
+// Configured by the daemon at /load with the host → service table; a
+// project with no `coverage: { browser: true }` service never attaches
+// anything, so it costs nothing.
+
+import { createHash } from "node:crypto";
+import type { CDPSession } from "playwright-core";
+import {
+  type DecodedSourceMap,
+  type FileCounts,
+  decodeSourceMap,
+  lcovDocument,
+  mapToOriginal,
+  mergeFileCounts,
+  rangesToLineCounts,
+  type V8FunctionCoverage,
+} from "./harness/browser-coverage.js";
+import { rawFetch } from "./harness/raw-fetch.js";
+
+export interface BrowserCoverageConfig {
+  /** Which covered service (if any) serves scripts from this host. */
+  serviceForHost(host: string): string | undefined;
+}
+
+let CONFIG: BrowserCoverageConfig | null = null;
+
+/** Set (or clear, with `null`) at /load. */
+export function configureBrowserCoverage(config: BrowserCoverageConfig | null): void {
+  CONFIG = config;
+}
+
+export function browserCoverageActive(): boolean {
+  return CONFIG !== null;
+}
+
+/** service → file → line → count, summed over every harvest. */
+const TOTALS = new Map<string, FileCounts>();
+
+interface ScriptInfo {
+  url: string;
+  sourceMapURL?: string;
+  source?: string;
+  /** `undefined` = not looked up yet; `null` = none / unusable. */
+  map?: DecodedSourceMap | null;
+}
+
+interface Collector {
+  cdp: CDPSession;
+  scripts: Map<string, ScriptInfo>;
+  inFlight: Promise<void> | null;
+}
+
+const COLLECTORS = new WeakMap<CDPSession, Collector>();
+
+/** Decoded maps by URL (or content hash for inline `data:` maps). */
+const SOURCE_MAPS = new Map<string, DecodedSourceMap | null>();
+
+const SOURCE_MAP_FETCH_TIMEOUT_MS = 10_000;
+
+/** Turn coverage on for a page's CDP session. Idempotent; a no-op when no
+ *  service opted in. */
+export async function attachBrowserCoverage(cdp: CDPSession): Promise<void> {
+  if (!CONFIG || COLLECTORS.has(cdp)) return;
+  const collector: Collector = { cdp, scripts: new Map(), inFlight: null };
+  COLLECTORS.set(cdp, collector);
+  cdp.on("Debugger.scriptParsed", (ev: { scriptId: string; url: string; sourceMapURL?: string }) => {
+    if (!ev.url) return;
+    collector.scripts.set(ev.scriptId, {
+      url: ev.url,
+      sourceMapURL: ev.sourceMapURL || undefined,
+    });
+  });
+  await cdp.send("Debugger.enable");
+  await cdp.send("Profiler.enable");
+  await cdp.send("Profiler.startPreciseCoverage", { callCount: true, detailed: true });
+}
+
+/**
+ * Take the session's counters and fold them into the totals. Concurrent
+ * calls coalesce onto the in-flight one. Errors (a page mid-navigation,
+ * a closed view) are swallowed: the counters then stay in V8 for the next
+ * take, nothing is lost.
+ */
+export async function harvestBrowserCoverage(cdp: CDPSession): Promise<void> {
+  const c = COLLECTORS.get(cdp);
+  if (!c || !CONFIG) return;
+  if (c.inFlight) return c.inFlight;
+  c.inFlight = harvestInner(c, CONFIG).catch(() => {}).finally(() => {
+    c.inFlight = null;
+  });
+  return c.inFlight;
+}
+
+async function harvestInner(c: Collector, config: BrowserCoverageConfig): Promise<void> {
+  const { result } = (await c.cdp.send("Profiler.takePreciseCoverage")) as {
+    result: { scriptId: string; url: string; functions: V8FunctionCoverage[] }[];
+  };
+  for (const script of result) {
+    const info = c.scripts.get(script.scriptId) ?? (script.url ? { url: script.url } : undefined);
+    if (!info) continue;
+    let url: URL;
+    try {
+      url = new URL(info.url);
+    } catch {
+      continue;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") continue;
+    const service = config.serviceForHost(url.hostname);
+    if (!service) continue;
+    if (info.source === undefined) {
+      try {
+        const r = (await c.cdp.send("Debugger.getScriptSource", {
+          scriptId: script.scriptId,
+        })) as { scriptSource: string };
+        info.source = r.scriptSource;
+      } catch {
+        continue; // script gone with its document
+      }
+      c.scripts.set(script.scriptId, info);
+    }
+    const generated = rangesToLineCounts(info.source, script.functions);
+    if (generated.size === 0) continue;
+    if (info.map === undefined) info.map = await sourceMapFor(info, url);
+    const files: FileCounts = info.map
+      ? mapToOriginal(info.map, generated)
+      : new Map([[url.pathname, generated]]);
+    let totals = TOTALS.get(service);
+    if (!totals) {
+      totals = new Map();
+      TOTALS.set(service, totals);
+    }
+    mergeFileCounts(totals, files);
+  }
+}
+
+/** Resolve, fetch (or inline-decode) and decode a script's source map. */
+async function sourceMapFor(info: ScriptInfo, scriptUrl: URL): Promise<DecodedSourceMap | null> {
+  let ref = info.sourceMapURL;
+  if (!ref && info.source) {
+    const m = /\/\/[#@]\s*sourceMappingURL=(\S+)\s*$/.exec(info.source.slice(-4096));
+    if (m) ref = m[1];
+  }
+  if (!ref) return null;
+  if (ref.startsWith("data:")) {
+    const key = "data:" + createHash("sha1").update(ref).digest("hex");
+    const cached = SOURCE_MAPS.get(key);
+    if (cached !== undefined) return cached;
+    const map = decodeSourceMap(parseDataUrl(ref));
+    SOURCE_MAPS.set(key, map);
+    return map;
+  }
+  let mapUrl: string;
+  try {
+    mapUrl = new URL(ref, scriptUrl).toString();
+  } catch {
+    return null;
+  }
+  const cached = SOURCE_MAPS.get(mapUrl);
+  if (cached !== undefined) return cached;
+  let map: DecodedSourceMap | null = null;
+  try {
+    const res = await rawFetch(mapUrl, { signal: AbortSignal.timeout(SOURCE_MAP_FETCH_TIMEOUT_MS) });
+    if (res.ok) map = decodeSourceMap(await res.json());
+  } catch {
+    map = null;
+  }
+  SOURCE_MAPS.set(mapUrl, map);
+  return map;
+}
+
+function parseDataUrl(ref: string): unknown {
+  const comma = ref.indexOf(",");
+  if (comma < 0) return null;
+  const meta = ref.slice(5, comma);
+  const payload = ref.slice(comma + 1);
+  try {
+    const text = /;base64$/i.test(meta)
+      ? Buffer.from(payload, "base64").toString("utf8")
+      : decodeURIComponent(payload);
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+/** One lcov document per service that has any browser coverage. */
+export function browserCoverageReports(): Map<string, string> {
+  const out = new Map<string, string>();
+  for (const [service, files] of TOTALS) {
+    if (files.size > 0) out.set(service, lcovDocument(files));
+  }
+  return out;
+}

package/src/browser.ts

@@ -38,6 +38,11 @@ import { recordBrowser, reserveBackdated, reserveEvent, truncateUtf8 } from "./r
 import { wrap } from "./inspect.js";
 import type { Wrapped } from "./inspect.js";
 import { describeUrlPattern, matchesUrl } from "./url-match.js";
+import {
+  attachBrowserCoverage,
+  browserCoverageActive,
+  harvestBrowserCoverage,
+} from "./browser-coverage.js";
 import type { UrlPattern } from "./url-match.js";
 export type { UrlPattern } from "./url-match.js";
 import {
@@ -1223,6 +1228,13 @@ async function spawnPage(context: BrowserContext): Promise<SpawnedPage> {
       console.warn("[spectest] failed to install rrweb recorder:", err);
     }
   }
+  // Frontend coverage (opt-in): turn on V8 precise coverage for this page.
+  // A no-op unless a service declared `coverage: { browser: true }`.
+  try {
+    await attachBrowserCoverage(cdp);
+  } catch {
+    /* coverage is best-effort; a page without it still works */
+  }
   return { page, cdp, recordingInstalled };
 }
 
@@ -1558,6 +1570,13 @@ async function rebuildView(holder: ViewHolder): Promise<void> {
   } catch {
     /* page may already be gone */
   }
+  // The old renderer's coverage counters die with its page; harvest what
+  // it had before the swap (rebuild is triggered on a fresh navigation).
+  try {
+    await harvestBrowserCoverage(holder.cdp);
+  } catch {
+    /* best-effort */
+  }
   const fresh = await spawnPage(holder.context);
   holder.page = fresh.page;
   holder.cdp = fresh.cdp;
@@ -1660,6 +1679,16 @@ function buildBackend(
         durationMs: endT - t,
       }, resv);
       await drain(action);
+      // Fold this page's V8 counters into the per-service totals now — a
+      // navigation to another origin replaces the renderer and its
+      // counters, so harvesting per op is what keeps them.
+      if (browserCoverageActive()) {
+        try {
+          await harvestBrowserCoverage(holder.cdp);
+        } catch {
+          /* best-effort */
+        }
+      }
       // Reads (`opts.wrap`) return user-visible JS values someone is likely to
       // assert on — provenance-wrap so `expect(...)` nests under this step.
       // Actions return void/internals; leave them raw to avoid Proxy surprises.
@@ -1688,6 +1717,13 @@ function buildBackend(
     if (recordingEnded) return;
     // Final drain before we stop writing to this recorder.
     await drain("close");
+    if (browserCoverageActive()) {
+      try {
+        await harvestBrowserCoverage(holder.cdp);
+      } catch {
+        /* best-effort */
+      }
+    }
     recordingEnded = true;
   }