@empiricalrun/playwright-utils 0.58.3 → 0.58.5

package/upstream-oracle/run.mjs

@@ -9,15 +9,17 @@
  * format. The fixture patch (intermediate-mode-ibr.patch) lets
  * PWTEST_IBR_ADAPTER swap IBR in as that reporter.
  *
- * Both modes run the same curated spec files; environment noise cancels in
- * the baseline diff. Any IBR-only failure fails the run — the job stays red
- * until IBR reaches parity (known gaps today: onStdIO, onError, result-level
- * annotations, attachment bodies). It turning green is the signal that the
- * built-in blob reporter is safe to drop.
+ * Both modes run the same spec files; environment noise cancels in the
+ * baseline diff. Any IBR-only failure fails the run — the job stays red
+ * until IBR reaches parity on these specs. It staying green is the signal
+ * that the built-in blob reporter is safe to drop. The one carve-out is
+ * EXPECTED_IBR_ONLY below: failures caused by injecting an external reporter
+ * rather than by the blob format, each with the reason it cannot pass, and
+ * each policed by a staleness check so the list cannot rot.
  *
  * Usage:
  *   node run.mjs --pw-repo <upstream clone at the pinned tag>
- *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N]
+ *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N] [--shard i/n]
  *
  * The clone must be bootstrapped (npm ci + npm run build). This script
  * (re)applies the fixture patch idempotently on every run.
@@ -31,18 +33,111 @@ import { fileURLToPath } from "node:url";
 
 const ORACLE_DIR = path.dirname(fileURLToPath(import.meta.url));
 
-// Curated, node-only (no browser) runner specs with high report-content
-// signal. stdio.spec.ts is the direct probe for the known onStdIO gap.
-const DEFAULT_SPECS = [
-  "basic.spec.ts",
-  "hooks.spec.ts",
-  "max-failures.spec.ts",
-  "stdio.spec.ts",
-  "test-modifiers.spec.ts",
-  "test-serial.spec.ts",
-  "test-step.spec.ts",
+// The default sweep is EVERY spec in upstream's tests/playwright-test —
+// IBR is meant to become the default blob reporter, so it has to stand the
+// whole runner suite. Only structurally out-of-scope specs are excluded,
+// each with its reason printed at startup (no silent caps). First matching
+// rule wins.
+const EXCLUSION_RULES = [
+  [
+    (name) => name.startsWith("playwright.ct-"),
+    "component-testing build; no runner-report signal",
+  ],
+  [
+    (name) => name.startsWith("playwright."),
+    "drives real browsers (not installed in the oracle env)",
+  ],
+  [
+    (name) => name.startsWith("ui-mode-"),
+    "UI mode; needs trace viewer + browsers",
+  ],
+  [
+    (name) =>
+      name === "to-have-screenshot.spec.ts" ||
+      name === "aria-snapshot-file.spec.ts" ||
+      name === "update-aria-snapshot.spec.ts",
+    "browser screenshots/aria snapshots",
+  ],
+  [
+    (name) => name === "types.spec.ts" || name === "types-2.spec.ts",
+    "tsc-only; exercises no runner report",
+  ],
+  [
+    (name) => name === "reporter-html.spec.ts",
+    "browser-driven HTML-report UI tests; no blob-format signal (merge " +
+      "semantics are covered by reporter-blob.spec.ts) and they busy-loop " +
+      "workers under load, wedging the sweep",
+  ],
+  // The two specs below are SLOW, not wedged — measured, they make steady
+  // progress and simply do not fit in SPEC_TIMEOUT_MS. The oracle runs one
+  // spec per invocation and Playwright keeps a file's tests on a single
+  // worker (fullyParallel defaults to false), so a long file runs strictly
+  // serially however many --workers it is given. Both were already
+  // contributing no signal, just a 10-minute timeout each.
+  [
+    (name) => name === "only-changed.spec.ts",
+    "git-based run selection (--only-changed), not report format; 13 tests " +
+      "that each shell out to git take ~13min serially here",
+  ],
+  [
+    (name) => name === "web-server.spec.ts",
+    "webServer config and lifecycle, not report format; 39 tests that each " +
+      "boot and tear down a server take longer still",
+  ],
 ];
 
+// Upstream tests that fail through IBR for reasons OUTSIDE the blob format —
+// artifacts of injecting an external reporter into upstream's harness, not
+// gaps in what IBR writes. They are reported and subtracted from the failure
+// count instead of turning the job red.
+//
+// Kept honest by the staleness check at the end: an entry whose spec ran and
+// did NOT fail is reported and fails the run, so a fixed test cannot sit here
+// pretending to be unfixable. Add an entry only with the reason it can never
+// pass; anything that could be fixed belongs in the failure list.
+const EXPECTED_IBR_ONLY = [
+  {
+    key:
+      "cache.spec.ts:53 › should automatically clean cached versions of a " +
+      "changed file",
+    reason:
+      "asserts the transform cache holds exactly one directory; loading IBR " +
+      "from a path runs its bundle through the same transform pipeline, so " +
+      "the cache gains a directory per module (32 observed). The built-in " +
+      "blob reporter ships inside Playwright and never enters that cache.",
+  },
+];
+
+/** Spec file name out of a failure key ("cache.spec.ts:53 › title"). */
+function specOfKey(key) {
+  return key.slice(0, key.indexOf(":"));
+}
+
+// Tests skipped by title via --grep-invert. Reserved for tests that WEDGE
+// the sweep — a busy-looped worker no per-test timeout can interrupt — so
+// exclusion is surgical (exact title match) and every entry carries the
+// observed reason. Everything else in the spec still runs.
+const HANGING_TESTS = [];
+
+function discoverSpecs(pwRepo) {
+  const specDir = path.join(pwRepo, "tests", "playwright-test");
+  const all = fs
+    .readdirSync(specDir)
+    .filter((f) => f.endsWith(".spec.ts"))
+    .sort();
+  const included = [];
+  const excluded = [];
+  for (const name of all) {
+    const rule = EXCLUSION_RULES.find(([match]) => match(name));
+    if (rule) {
+      excluded.push([name, rule[1]]);
+    } else {
+      included.push(name);
+    }
+  }
+  return { included, excluded };
+}
+
 const ADAPTER_PATH = path.join(ORACLE_DIR, "ibr-adapter.js");
 const PATCH_PATH = path.join(ORACLE_DIR, "intermediate-mode-ibr.patch");
 const DEFAULT_IBR_DIST = path.join(
@@ -51,7 +146,7 @@ const DEFAULT_IBR_DIST = path.join(
 );
 
 function parseArgs(argv) {
-  const args = { specs: DEFAULT_SPECS, workers: 4 };
+  const args = { specs: null, workers: 4, shard: null };
   // Normalize --flag=value into --flag value.
   argv = argv.flatMap((a) =>
     a.startsWith("--") && a.includes("=")
@@ -66,6 +161,15 @@ function parseArgs(argv) {
       args.specs = argv[++i].split(",").filter(Boolean);
     } else if (arg === "--workers") {
       args.workers = Number(argv[++i]);
+    } else if (arg === "--shard") {
+      const match = /^(\d+)\/(\d+)$/.exec(argv[++i] ?? "");
+      if (!match) {
+        throw new Error("--shard expects i/n (e.g. --shard 2/4)");
+      }
+      args.shard = { index: Number(match[1]), total: Number(match[2]) };
+      if (args.shard.index < 1 || args.shard.index > args.shard.total) {
+        throw new Error(`--shard index out of range: ${argv[i]}`);
+      }
     } else {
       throw new Error(`Unknown argument: ${arg}`);
     }
@@ -134,7 +238,19 @@ async function applyFixturePatch(pwRepo) {
   }
 }
 
-async function runMode({ pwRepo, specs, workers, mode, ibrDist }) {
+// Hard cap per spec-mode run (milliseconds — Playwright's --global-timeout
+// unit). Some upstream tests can busy-loop a worker under load (sync string
+// processing that no per-test timeout can interrupt — the timeouts run on
+// the same blocked event loop); which test wedges is timing-dependent, so
+// title exclusion cannot converge. Playwright's global timeout fires from
+// the main process and SIGKILLs through busy loops. Each spec runs as its
+// own bounded invocation, so a wedge costs one spec's cap, names the spec,
+// and the sweep continues — hangs become inventory instead of aborting the
+// whole mode.
+const SPEC_TIMEOUT_MS = 10 * 60 * 1_000;
+
+async function runMode({ pwRepo, spec, workers, mode, ibrDist }) {
+  const startedAt = Date.now();
   const jsonOut = path.join(
     fs.mkdtempSync(path.join(os.tmpdir(), "ibr-oracle-")),
     `${mode}.json`,
@@ -149,32 +265,68 @@ async function runMode({ pwRepo, specs, workers, mode, ibrDist }) {
     env.PWTEST_IBR_ADAPTER = ADAPTER_PATH;
     env.PWTEST_IBR_DIST = ibrDist;
   }
+  // The inner run writes its JSON report at the end, so nothing streams to
+  // stdout while it runs. Heartbeat from the test-results dir (the outer
+  // harness creates one dir per executed test) so long modes show progress.
+  const testResultsDir = path.join(pwRepo, "test-results");
+  const heartbeat = setInterval(() => {
+    let dirs = 0;
+    try {
+      dirs = fs.readdirSync(testResultsDir).length;
+    } catch {
+      // Not created yet.
+    }
+    const minutes = Math.round((Date.now() - startedAt) / 60_000);
+    console.log(
+      `  [${mode} ${spec}] ~${dirs} tests executed, ${minutes}m elapsed`,
+    );
+  }, 60_000);
   const { code, output } = await run(
     "npm",
     [
       "run",
       "ttest",
       "--",
-      ...specs,
+      spec,
       `--workers=${workers}`,
       // Absorbs outer-harness flake: a scenario that fails on IBR's real gaps
       // fails every retry, while an infra hiccup passes on retry and is
       // reported "flaky" (not counted as a failure).
       "--retries=2",
+      `--global-timeout=${SPEC_TIMEOUT_MS}`,
+      // An empty --grep-invert would match (and thus drop) every test, so
+      // the flag only appears when there are titles to skip.
+      ...(HANGING_TESTS.length
+        ? [`--grep-invert=${HANGING_TESTS.map((h) => h.pattern).join("|")}`]
+        : []),
       "--reporter=json",
     ],
     { cwd: pwRepo, env },
   );
+  clearInterval(heartbeat);
+  const elapsedSeconds = Math.round((Date.now() - startedAt) / 1_000);
+  // A globally-timed-out run still writes a valid JSON report with the
+  // remaining tests marked "did not run", which would otherwise diff as a
+  // false green — report the hang instead of trusting the partial report.
+  if (/Timed out waiting .* for the test suite/.test(output)) {
+    return { hung: true, elapsedSeconds };
+  }
   let report;
   try {
     report = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
   } catch {
     console.error(output.slice(-4000));
     throw new Error(
-      `${mode}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
+      `${mode} ${spec}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
     );
   }
-  return collectFailures(report);
+  const stats = report.stats ?? {};
+  const totalRun =
+    (stats.expected ?? 0) +
+    (stats.unexpected ?? 0) +
+    (stats.flaky ?? 0) +
+    (stats.skipped ?? 0);
+  return { failures: collectFailures(report), totalRun, elapsedSeconds };
 }
 
 const args = parseArgs(process.argv.slice(2));
@@ -188,41 +340,130 @@ if (!fs.existsSync(ibrDist)) {
 
 await applyFixturePatch(pwRepo);
 
+if (!args.specs) {
+  const { included, excluded } = discoverSpecs(pwRepo);
+  args.specs = included;
+  console.log(`Excluded specs (${excluded.length}):`);
+  for (const [name, reason] of excluded) {
+    console.log(`  [excluded] ${name} — ${reason}`);
+  }
+}
+if (args.shard) {
+  // Round-robin slice so slow specs spread across shards.
+  args.specs = args.specs.filter(
+    (_, i) => i % args.shard.total === args.shard.index - 1,
+  );
+  console.log(
+    `Shard ${args.shard.index}/${args.shard.total}: ${args.specs.length} specs`,
+  );
+}
+
 console.log(`Specs: ${args.specs.join(", ")}`);
-console.log("Running baseline (built-in blob reporter)...");
-const blobFailures = await runMode({
-  pwRepo,
-  specs: args.specs,
-  workers: args.workers,
-  mode: "blob",
-});
-console.log(
-  `Baseline failures (env noise, cancelled in diff): ${blobFailures.size}`,
-);
-for (const key of [...blobFailures].sort()) {
-  console.log(`  [baseline] ${key}`);
+for (const { pattern, reason } of HANGING_TESTS) {
+  console.log(`  [skipped-by-title] /${pattern}/ — ${reason}`);
+}
+for (const { key, reason } of EXPECTED_IBR_ONLY) {
+  console.log(`  [expected-ibr-only] ${key} — ${reason}`);
 }
+// Each spec runs both modes back to back as its own bounded invocation
+// (see SPEC_TIMEOUT_MS); a hang or count mismatch quarantines that spec and
+// the sweep continues.
+const ibrOnly = [];
+const hungSpecs = [];
+const mismatchedSpecs = [];
+const expectedByKey = new Map(EXPECTED_IBR_ONLY.map((e) => [e.key, e]));
+const matchedExpected = new Set();
+const comparedSpecs = new Set();
+let baselineNoise = 0;
 
-console.log("Running through IBR adapter...");
-const ibrFailures = await runMode({
-  pwRepo,
-  specs: args.specs,
-  workers: args.workers,
-  mode: "ibr",
-  ibrDist,
-});
+for (const spec of args.specs) {
+  const blob = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "blob",
+  });
+  if (blob.hung) {
+    console.log(`  [hung] ${spec} (blob mode, ${blob.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (blob mode)`);
+    continue;
+  }
+  const ibr = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "ibr",
+    ibrDist,
+  });
+  if (ibr.hung) {
+    console.log(`  [hung] ${spec} (ibr mode, ${ibr.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (ibr mode)`);
+    continue;
+  }
+  if (blob.totalRun === 0 || blob.totalRun !== ibr.totalRun) {
+    console.log(
+      `  [mismatch] ${spec}: blob ran ${blob.totalRun}, ibr ran ${ibr.totalRun}`,
+    );
+    mismatchedSpecs.push(spec);
+    continue;
+  }
+  comparedSpecs.add(spec);
+  const specIbrOnly = [...ibr.failures]
+    .filter((key) => !blob.failures.has(key))
+    .sort();
+  baselineNoise += blob.failures.size;
+  const specExpected = specIbrOnly.filter((key) => expectedByKey.has(key));
+  for (const key of specExpected) {
+    matchedExpected.add(key);
+  }
+  const specUnexpected = specIbrOnly.filter((key) => !expectedByKey.has(key));
+  ibrOnly.push(...specUnexpected);
+  console.log(
+    `  ${spec}: ${blob.totalRun} tests, ` +
+      `${blob.failures.size} baseline noise, ${specUnexpected.length} ibr-only` +
+      (specExpected.length ? `, ${specExpected.length} expected` : "") +
+      ` (blob ${blob.elapsedSeconds}s, ibr ${ibr.elapsedSeconds}s)`,
+  );
+}
 
-const ibrOnly = [...ibrFailures].filter((key) => !blobFailures.has(key)).sort();
+console.log(
+  `\nBaseline failures (env noise, cancelled in diff): ${baselineNoise}`,
+);
+console.log(
+  `Expected IBR-only failures (not counted): ${matchedExpected.size}`,
+);
 console.log(`IBR-only failures: ${ibrOnly.length}`);
-for (const key of ibrOnly) {
+for (const key of ibrOnly.sort()) {
   console.log(`  [ibr-only] ${key}`);
 }
+for (const spec of hungSpecs) {
+  console.log(`  [hung] ${spec}`);
+}
+for (const spec of mismatchedSpecs) {
+  console.log(`  [mismatch] ${spec}`);
+}
+
+// An expected failure whose spec ran and passed means the list is stale —
+// that is a fix to celebrate by deleting the entry, not a silent pass.
+const staleExpected = EXPECTED_IBR_ONLY.filter(
+  ({ key }) => comparedSpecs.has(specOfKey(key)) && !matchedExpected.has(key),
+);
+for (const { key } of staleExpected) {
+  console.log(`  [stale-expected] ${key} — now passes; drop it from the list`);
+}
 
-if (ibrOnly.length > 0) {
+if (
+  ibrOnly.length > 0 ||
+  hungSpecs.length > 0 ||
+  mismatchedSpecs.length > 0 ||
+  staleExpected.length > 0
+) {
   console.error(
-    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR. ` +
-      `The job stays red until IBR reaches blob parity (known gaps: onStdIO, ` +
-      `onError, result-level annotations, attachment bodies).`,
+    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR, ` +
+      `${hungSpecs.length} spec run(s) hung, ${mismatchedSpecs.length} ` +
+      `spec(s) mismatched, ${staleExpected.length} stale expected-failure ` +
+      `entr${staleExpected.length === 1 ? "y" : "ies"}. The job stays red ` +
+      `until IBR reaches blob parity on these specs.`,
   );
   process.exit(1);
 }