@specific.dev/spectest 0.57.0 → 0.59.0

package/src/coverage.test.ts

@@ -0,0 +1,192 @@
+import { describe, expect, test } from "bun:test";
+import { spawn, type ChildProcess } from "node:child_process";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+  NODE_COVERAGE_HOOK,
+  NODE_COVERAGE_HOOK_PATH,
+  NODE_COVERAGE_SOCKET,
+  applyCoverageAdapters,
+  appendEnvFlag,
+  browser,
+  command,
+  node,
+  validateCoverage,
+  type CoverageCaptureContext,
+} from "./coverage.js";
+import type { ServiceConfig } from "./index.js";
+
+const base: ServiceConfig = {
+  image: { type: "registry", reference: "node:20" },
+  command: "node server.js",
+  env: { NODE_OPTIONS: "--max-old-space-size=512" },
+};
+
+describe("validateCoverage", () => {
+  test("accepts true, a list of adapters, nothing else", () => {
+    validateCoverage("a", undefined);
+    validateCoverage("a", true);
+    validateCoverage("a", { adapters: [node(), { name: "x" }] });
+    expect(() => validateCoverage("a", false)).toThrow(/invalid `coverage` value/);
+    expect(() => validateCoverage("a", { command: "x" })).toThrow(/invalid `coverage` value/);
+    expect(() => validateCoverage("a", [node()])).toThrow(/invalid `coverage` value/);
+    expect(() => validateCoverage("a", { adapters: [{ name: "" }] })).toThrow(/invalid coverage adapter/);
+    expect(() => validateCoverage("a", { adapters: [{ name: "x", capture: 1 }] })).toThrow(/invalid coverage adapter/);
+  });
+});
+
+describe("applyCoverageAdapters", () => {
+  test("node: appends NODE_OPTIONS, sets NODE_V8_COVERAGE, mounts the hook", () => {
+    const out = applyCoverageAdapters("api", { ...base, coverage: { adapters: [node()] } });
+    expect(out.env?.NODE_OPTIONS).toBe(`--max-old-space-size=512 --require ${NODE_COVERAGE_HOOK_PATH}`);
+    expect(out.env?.NODE_V8_COVERAGE).toBe("/spectest/coverage");
+    expect(out.files?.map((f) => f.path)).toEqual([NODE_COVERAGE_HOOK_PATH]);
+    expect(out.files?.[0]?.content).toBe(NODE_COVERAGE_HOOK);
+  });
+  test("the adapters stay on the config for the harness, and serialize as names", () => {
+    const out = applyCoverageAdapters("api", { ...base, coverage: { adapters: [node(), browser()] } });
+    const list = (out.coverage as { adapters: readonly { name: string; capture?: unknown; load?: unknown }[] }).adapters;
+    expect(list.map((a) => a.name)).toEqual(["node", "browser"]);
+    expect(typeof list[0]?.capture).toBe("function");
+    expect(typeof list[1]?.load).toBe("function");
+    expect(JSON.parse(JSON.stringify(out)).coverage).toEqual({
+      adapters: [{ adapter: "node" }, { adapter: "browser" }],
+    });
+  });
+  test("true and undefined pass through untouched", () => {
+    const t = { ...base, coverage: true as const };
+    expect(applyCoverageAdapters("api", t)).toBe(t);
+    expect(applyCoverageAdapters("api", base)).toBe(base);
+  });
+  test("appendEnvFlag sets when absent", () => {
+    expect(appendEnvFlag(undefined, "X", "a")).toEqual({ X: "a" });
+    expect(appendEnvFlag({ X: " b " }, "X", "a")).toEqual({ X: "b a" });
+  });
+  test("coverage.command refuses an empty command", () => {
+    expect(() => command("  ")).toThrow(/non-empty/);
+  });
+});
+
+/** A capture context over a real directory, no container. */
+function fakeCtx(dir: string, signal = new AbortController().signal): CoverageCaptureContext {
+  return {
+    service: "api",
+    reportDir: dir,
+    signal,
+    async exec() {
+      throw new Error("no container");
+    },
+    async writeReport(name, content) {
+      await fs.writeFile(path.join(dir, name), content);
+    },
+  };
+}
+
+describe("browserCoverage", () => {
+  test("writes an empty lcov when nothing ran for the service", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-cov-"));
+    await browser().capture!(fakeCtx(dir));
+    expect(await fs.readFile(path.join(dir, "browser.lcov"), "utf8")).toBe("TN:\n");
+  });
+});
+
+// The node hook against a real `node` (the box has one; skipped where it
+// does not). A long-lived process armed with the hook must answer the
+// socket with a fresh V8 report; a second process must stay a bystander
+// and exit on its own; the socket path must work across a bind mount —
+// which is what a plain directory path stands in for here.
+const hasNode = await new Promise<boolean>((resolve) => {
+  const p = spawn("node", ["--version"]);
+  p.on("error", () => resolve(false));
+  p.on("exit", (code) => resolve(code === 0));
+});
+
+describe.if(hasNode)("nodeCoverage hook (real node)", () => {
+  async function startServer(dir: string): Promise<ChildProcess> {
+    const hook = path.join(dir, "hook.cjs");
+    await fs.writeFile(hook, NODE_COVERAGE_HOOK);
+    const server = path.join(dir, "server.js");
+    await fs.writeFile(server, "setInterval(() => {}, 1000); process.stdout.write('up\\n');\n");
+    const child = spawn("node", ["--require", hook, server], {
+      env: { ...process.env, NODE_V8_COVERAGE: dir },
+      stdio: ["ignore", "pipe", "inherit"],
+    });
+    await new Promise<void>((resolve) => child.stdout!.once("data", () => resolve()));
+    // The socket is bound after the module graph runs; wait for it.
+    const sock = path.join(dir, NODE_COVERAGE_SOCKET);
+    for (let i = 0; i < 100; i++) {
+      try {
+        await fs.stat(sock);
+        break;
+      } catch {
+        await new Promise((r) => setTimeout(r, 20));
+      }
+    }
+    return child;
+  }
+
+  test("capture asks the live server for a dump; a bystander exits on its own", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-covhook-"));
+    const server = await startServer(dir);
+    try {
+      const reportsBefore = (await fs.readdir(dir)).filter((n) => n.startsWith("coverage-"));
+      expect(reportsBefore).toEqual([]);
+      await node().capture!(fakeCtx(dir));
+      const reports = (await fs.readdir(dir)).filter((n) => n.startsWith("coverage-"));
+      expect(reports.length).toBe(1);
+      const doc = JSON.parse(await fs.readFile(path.join(dir, reports[0]!), "utf8")) as {
+        result: { url: string }[];
+      };
+      expect(doc.result.some((s) => s.url.endsWith("/server.js"))).toBe(true);
+
+      // A second process with the same env: the socket is held, so it must
+      // not try to serve, and it must exit (the server's socket is unref'd,
+      // so a bystander is never kept alive by it).
+      const bystander = spawn("node", ["--require", path.join(dir, "hook.cjs"), "-e", "1"], {
+        env: { ...process.env, NODE_V8_COVERAGE: dir },
+        stdio: "ignore",
+      });
+      const code = await new Promise<number | null>((resolve) => bystander.on("exit", resolve));
+      expect(code).toBe(0);
+      // …and it wrote its own exit-time report, as every node process does.
+      const after = (await fs.readdir(dir)).filter((n) => n.startsWith("coverage-"));
+      expect(after.length).toBe(2);
+
+      // The live server still answers.
+      await node().capture!(fakeCtx(dir));
+      expect((await fs.readdir(dir)).filter((n) => n.startsWith("coverage-")).length).toBe(3);
+    } finally {
+      server.kill("SIGKILL");
+    }
+  });
+
+  test("a stale socket left by a dead process is reclaimed", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-covhook-"));
+    const first = await startServer(dir);
+    first.kill("SIGKILL");
+    await new Promise<void>((resolve) => first.on("exit", () => resolve()));
+    expect(await fs.stat(path.join(dir, NODE_COVERAGE_SOCKET))).toBeTruthy(); // stale file
+    const second = await startServer(dir);
+    try {
+      // Give the EADDRINUSE → probe → unlink → listen dance a moment.
+      let ok = false;
+      for (let i = 0; i < 50 && !ok; i++) {
+        try {
+          await node().capture!(fakeCtx(dir));
+          ok = true;
+        } catch {
+          await new Promise((r) => setTimeout(r, 50));
+        }
+      }
+      expect(ok).toBe(true);
+    } finally {
+      second.kill("SIGKILL");
+    }
+  });
+
+  test("no server: capture fails naming the socket", async () => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "spectest-covhook-"));
+    await expect(node().capture!(fakeCtx(dir))).rejects.toThrow(/no node process is serving/);
+  });
+});

package/src/coverage.ts

@@ -0,0 +1,327 @@
+// Coverage adapters — how spectest gets a coverage report out of a service.
+//
+// A service opts in with `coverage: true` (the program writes its own
+// reports into `/spectest/coverage/`) or `coverage: { adapters: [...] }`
+// (spectest gets the reports out). An adapter has three optional moments:
+//
+//   configure  — config time, inside `defineEnvironment`. Rewrites the
+//                service (env, files, command). Pure: its output is part
+//                of the wire config and of the warm-template hash.
+//   load       — harness, at /load. Asks the harness for capabilities a
+//                container cannot supply itself (browser scripts).
+//   capture    — harness, after bring-up and after every test. Must leave
+//                a report in the directory.
+//
+// After every adapter ran, the harness reads the directory the same way
+// it does for `coverage: true`: at least one well-formed report (lcov or
+// V8 JSON), shipped verbatim. Adapters convert nothing — the control
+// plane derives what it needs from the stored bytes.
+//
+// Imported from `@specific.dev/spectest/coverage`. The three shipped
+// adapters: `node()` (V8 JSON through a hook spectest mounts), `browser()`
+// (scripts the guest browser loads from the service), `command(cmd)` (run
+// a command, then read). A project writes its own with `defineAdapter`.
+//
+//   import * as coverage from "@specific.dev/spectest/coverage";
+//   coverage: { adapters: [coverage.node(), coverage.browser()] }
+
+import type { ServiceConfig } from "./index.js";
+import { browserCoverageReports } from "./browser-coverage.js";
+
+import { COVERAGE_CONTAINER_DIR } from "./harness/coverage.js";
+export { COVERAGE_CONTAINER_DIR };
+
+/** Where `node()` mounts its hook inside the container. */
+export const NODE_COVERAGE_HOOK_PATH = "/spectest/coverage-hook.cjs";
+
+/** The socket the node hook answers on, relative to the coverage dir. */
+export const NODE_COVERAGE_SOCKET = ".ctl";
+
+/** What `configure` learns about the service it rewrites. */
+export interface CoverageConfigureInfo {
+  /** The services-map key. */
+  key: string;
+}
+
+/** The harness capabilities an adapter can ask for at `/load`. */
+export interface CoverageLoadContext {
+  /** The services-map key. */
+  service: string;
+  /**
+   * Collect V8 coverage for every script the guest browser
+   * (`ctx.browser()` / `ctx.mobile()`) loads from this service's origins:
+   * its key, `<key>.internal`, its `hostnames`/`dnsName` aliases, its
+   * `tls`/`proxy` hostnames and any service-targeted wildcard. The
+   * harness maps each script back to source through the served source
+   * map and keeps per-service totals in memory (they fork with the
+   * environment). `browser()` calls this and writes the totals
+   * at capture.
+   */
+  collectBrowserScripts(): void;
+}
+
+/** What one adapter's `capture` runs against. */
+export interface CoverageCaptureContext {
+  /** The services-map key. */
+  service: string;
+  /**
+   * The coverage directory, seen from the harness. It is the same
+   * directory the container sees at `/spectest/coverage/` (a bind
+   * mount), so a file written here is in the container and a socket the
+   * container bound here is reachable.
+   */
+  reportDir: string;
+  /** Run a shell command (`sh -c`) inside the container. Rejects on a
+   *  non-zero exit with the command's output in the message. */
+  exec(command: string): Promise<{ stdout: string; stderr: string }>;
+  /** Write one report file (name relative to the directory). */
+  writeReport(name: string, content: string): Promise<void>;
+  /** Aborts when the per-service capture budget runs out. */
+  signal: AbortSignal;
+}
+
+export interface CoverageAdapter {
+  /** Names the adapter in errors (`coverage adapter "node" on service …`). */
+  name: string;
+  /**
+   * Rewrite the service at config time. Must be pure and deterministic
+   * (the result is hashed into the warm-template key) and must **append**
+   * to `env` values like `NODE_OPTIONS` rather than replace them.
+   */
+  configure?(service: ServiceConfig, info: CoverageConfigureInfo): ServiceConfig;
+  /** Ask the harness for capabilities, at `/load`. */
+  load?(ctx: CoverageLoadContext): void | Promise<void>;
+  /** Make the report appear in the directory. A throw fails the test. */
+  capture?(ctx: CoverageCaptureContext): void | Promise<void>;
+}
+
+/** `ServiceConfig.coverage`: the program writes its own reports, or a
+ *  list of adapters gets them out. */
+export type ServiceCoverage = true | { adapters: readonly CoverageAdapter[] };
+
+/** Type an inline adapter. Identity at runtime. */
+export function defineAdapter(adapter: CoverageAdapter): CoverageAdapter {
+  return adapter;
+}
+
+/** Check one service's `coverage` field. Exported so the daemon applies
+ *  the same rule to a runtime `startService` spec. */
+export function validateCoverage(service: string, cov: unknown): void {
+  if (cov === undefined || cov === true) return;
+  const adapters = (cov as { adapters?: unknown } | null)?.adapters;
+  if (typeof cov === "object" && cov !== null && Array.isArray(adapters)) {
+    for (const a of adapters) {
+      const ok =
+        typeof a === "object" &&
+        a !== null &&
+        typeof (a as CoverageAdapter).name === "string" &&
+        (a as CoverageAdapter).name.length > 0 &&
+        ["configure", "load", "capture"].every(
+          (k) =>
+            (a as Record<string, unknown>)[k] === undefined ||
+            typeof (a as Record<string, unknown>)[k] === "function",
+        );
+      if (!ok) {
+        throw new Error(
+          `service "${service}" has an invalid coverage adapter — an adapter is \`{ name, configure?, load?, capture? }\` (see \`defineAdapter\` in @specific.dev/spectest/coverage)`,
+        );
+      }
+    }
+    return;
+  }
+  throw new Error(
+    `service "${service}" has an invalid \`coverage\` value — use \`true\` (the program writes its own reports) or \`{ adapters: [...] }\` from @specific.dev/spectest/coverage (\`node()\`, \`browser()\`, \`command("…")\`)`,
+  );
+}
+
+/** The adapters of a `coverage` value (`true` has none). */
+export function coverageAdapters(cov: ServiceCoverage | undefined): readonly CoverageAdapter[] {
+  return typeof cov === "object" && cov !== null ? cov.adapters : [];
+}
+
+/**
+ * Apply every adapter's `configure` to a service, in list order. Called by
+ * `defineEnvironment` (after group expansion, before validation) and by
+ * the daemon for a runtime `startService` spec. The adapter list stays on
+ * the returned config for the harness; on the wire it serializes as the
+ * adapter names (`toJSON`), never as functions.
+ */
+export function applyCoverageAdapters<S extends ServiceConfig>(key: string, svc: S): S {
+  validateCoverage(key, svc.coverage);
+  const adapters = coverageAdapters(svc.coverage);
+  if (adapters.length === 0) return svc;
+  let out: ServiceConfig = svc;
+  for (const a of adapters) {
+    if (a.configure) out = a.configure(out, { key });
+  }
+  const wire = { adapters: adapters.map((a) => withWireName(a)) };
+  return { ...out, coverage: wire } as unknown as S;
+}
+
+function withWireName(a: CoverageAdapter): CoverageAdapter {
+  if (Object.prototype.hasOwnProperty.call(a, "toJSON")) return a;
+  return Object.assign(Object.create(Object.getPrototypeOf(a) as object | null), a, {
+    toJSON: () => ({ adapter: a.name }),
+  }) as CoverageAdapter;
+}
+
+/** `env` with `value` appended to `name` (space-separated), or set. */
+export function appendEnvFlag(
+  env: Readonly<Record<string, string>> | undefined,
+  name: string,
+  value: string,
+): Record<string, string> {
+  const prior = env?.[name]?.trim();
+  return { ...env, [name]: prior ? `${prior} ${value}` : value };
+}
+
+// ── node ──────────────────────────────────────────────────────────
+
+/**
+ * The hook `node()` mounts and `--require`s into every node
+ * process of the container. Node writes V8 coverage JSON into
+ * `NODE_V8_COVERAGE` when a process exits; a long-lived server never
+ * exits, so the hook binds a Unix socket in the coverage directory and
+ * calls `v8.takeCoverage()` for each connection. The first process to
+ * start owns the socket (a stale socket left by a dead process is
+ * reclaimed); a later process — a CLI run by `ctx.exec` — leaves it
+ * alone and writes at its own exit. No signal is used: signals are
+ * claimed by frameworks (SIGUSR2 stops a Temporal worker, restarts
+ * nodemon), a socket is nobody's.
+ */
+export const NODE_COVERAGE_HOOK = `"use strict";
+// spectest coverage hook (coverage.node() adapter). See \`spectest docs /services/coverage\`.
+if (process.env.NODE_V8_COVERAGE) {
+  const net = require("node:net");
+  const fs = require("node:fs");
+  const v8 = require("node:v8");
+  const SOCK = require("node:path").join(process.env.NODE_V8_COVERAGE, ${JSON.stringify(NODE_COVERAGE_SOCKET)});
+  const server = net.createServer((conn) => {
+    // A dump is taken only on an explicit "dump" request: a bystander's
+    // liveness probe connects and closes without one.
+    let buf = "";
+    conn.on("data", (chunk) => {
+      buf += chunk;
+      if (!buf.includes("\\n")) return;
+      let reply;
+      try {
+        if (!buf.startsWith("dump")) throw new Error("unknown request");
+        v8.takeCoverage();
+        reply = "ok\\n";
+      } catch (err) {
+        reply = "error " + (err && err.message ? err.message : String(err)) + "\\n";
+      }
+      conn.end(reply);
+    });
+  });
+  server.unref();
+  server.on("error", (err) => {
+    if (err.code !== "EADDRINUSE") return;
+    // Someone holds the socket. If it answers, it is the live server and
+    // this process is a bystander; if not, it is a stale file.
+    const probe = net.connect(SOCK);
+    probe.on("connect", () => probe.destroy());
+    probe.on("error", () => {
+      try { fs.unlinkSync(SOCK); } catch {}
+      server.listen(SOCK);
+    });
+  });
+  server.listen(SOCK);
+}
+`;
+
+/**
+ * Coverage for a Node service. Sets `NODE_V8_COVERAGE` to the coverage
+ * directory (every node process in the container then writes V8
+ * coverage JSON when it exits) and `--require`s a hook that lets spectest
+ * ask the long-lived server process for a dump at capture time. Nothing
+ * for the app to write. Source maps: with `--enable-source-maps` (or a
+ * `sourceMappingURL` next to the file) Node records the map in the
+ * report, which is what maps a TypeScript service back to its sources.
+ */
+export function node(): CoverageAdapter {
+  return {
+    name: "node",
+    configure(svc) {
+      const env = appendEnvFlag(svc.env, "NODE_OPTIONS", `--require ${NODE_COVERAGE_HOOK_PATH}`);
+      env.NODE_V8_COVERAGE = COVERAGE_CONTAINER_DIR;
+      return {
+        ...svc,
+        env,
+        files: [...(svc.files ?? []), { path: NODE_COVERAGE_HOOK_PATH, content: NODE_COVERAGE_HOOK }],
+      };
+    },
+    async capture(ctx) {
+      const { connect } = await import("node:net");
+      const path = await import("node:path");
+      const sock = path.join(ctx.reportDir, NODE_COVERAGE_SOCKET);
+      const reply = await new Promise<string>((resolve, reject) => {
+        const chunks: Buffer[] = [];
+        const c = connect(sock, () => c.write("dump\n"));
+        const onAbort = (): void => {
+          c.destroy();
+          reject(new Error("timed out waiting for the node hook to write a report"));
+        };
+        ctx.signal.addEventListener("abort", onAbort, { once: true });
+        c.on("data", (b: Buffer) => chunks.push(b));
+        c.on("error", (err: NodeJS.ErrnoException) => {
+          ctx.signal.removeEventListener("abort", onAbort);
+          reject(
+            new Error(
+              err.code === "ENOENT" || err.code === "ECONNREFUSED"
+                ? `no node process is serving ${COVERAGE_CONTAINER_DIR}/${NODE_COVERAGE_SOCKET} — is the service's main process node, and does it run with the service's env (NODE_OPTIONS)?`
+                : err.message,
+            ),
+          );
+        });
+        c.on("close", () => {
+          ctx.signal.removeEventListener("abort", onAbort);
+          resolve(Buffer.concat(chunks).toString("utf8").trim());
+        });
+      });
+      if (reply !== "ok") throw new Error(`the node hook answered: ${reply || "(nothing)"}`);
+    },
+  };
+}
+
+// ── browser ───────────────────────────────────────────────────────
+
+/**
+ * Coverage for the frontend a service serves. The code runs in the guest
+ * browser — spectest's own process — so no report can be written by the
+ * container; spectest collects V8 coverage for every script the browser
+ * loads from this service's origins, maps it back to source through the
+ * served source map, and writes one lcov per capture. Nothing ran yet on
+ * this branch ⇒ an lcov with no records, which is a report saying so.
+ */
+export function browser(): CoverageAdapter {
+  return {
+    name: "browser",
+    load(ctx) {
+      ctx.collectBrowserScripts();
+    },
+    async capture(ctx) {
+      const lcov = browserCoverageReports().get(ctx.service) ?? "TN:\n";
+      await ctx.writeReport("browser.lcov", lcov);
+    },
+  };
+}
+
+// ── command ───────────────────────────────────────────────────────
+
+/**
+ * Run `command` (via `sh -c`) in the container at every capture, so the
+ * service writes a fresh report into `/spectest/coverage/`. The escape
+ * hatch for a runtime with no shipped adapter.
+ */
+export function command(command: string): CoverageAdapter {
+  if (typeof command !== "string" || command.trim().length === 0) {
+    throw new Error("coverage.command: `command` must be a non-empty shell command");
+  }
+  return {
+    name: "command",
+    async capture(ctx) {
+      await ctx.exec(command);
+    },
+  };
+}