@empiricalrun/playwright-utils 0.58.1 → 0.58.3

package/upstream-oracle/README.md

@@ -0,0 +1,48 @@
+# Upstream Playwright oracle for the IncrementalBlobReporter
+
+Runs upstream Playwright's own runner test suite (`tests/playwright-test`)
+through our IncrementalBlobReporter (IBR) and diffs failures against the
+built-in blob reporter as baseline. Their
+`PWTEST_INTERMEDIATE_BLOB_REPORT=1` harness mode reroutes every inline
+scenario through `<reporter> -> merge-reports -> json` before applying the
+normal assertions — Playwright's own completeness check for the blob format.
+Swapping IBR in (via `ibr-adapter.js` + the one-line
+`intermediate-mode-ibr.patch`) turns it into a parity oracle for us.
+
+This backs the plan to drop the built-in blob reporter from sharded runs:
+before trusting IBR as the only report, its zip must merge identically on
+scenarios we did not think to write ourselves. The fast, per-PR half of that
+story is `src/reporter/blob-ibr-parity.test.ts`; this oracle is the deep
+audit, run by `.github/workflows/ibr-blob-oracle.yaml` (and re-run on every
+Playwright version-pin bump).
+
+## Files
+
+- `run.mjs` — orchestrator: applies the patch, runs the curated specs in both
+  modes, fails on any IBR-only failure.
+- `ibr-adapter.js` — bridges IBR's env/paths contract to the blob reporter's
+  (config.shard → SHARD_INDEX env, zip relocation to `blob-report/`).
+- `intermediate-mode-ibr.patch` — one-line upstream fixture change making the
+  intermediate reporter selectable via `PWTEST_IBR_ADAPTER`.
+
+The job is EXPECTED to stay red until IBR reaches parity: today ~55 upstream
+tests fail only through IBR, all tracing to the known gaps (`onStdIO`,
+`onError`, result-level annotations, attachment bodies — encoded as `it.fails`
+in `src/reporter/blob-ibr-parity.test.ts`). The failure count shrinking to
+zero is the signal that the built-in blob reporter is safe to drop.
+
+## Running locally
+
+```bash
+# One-time: clone + bootstrap upstream at the pinned tag (~10 min)
+git clone --depth 1 --branch v1.62.1 https://github.com/microsoft/playwright.git ~/playwright-upstream
+cd ~/playwright-upstream && npm ci && npm run build
+
+# Build IBR, then run the oracle
+cd <monorepo>/packages/playwright-utils && pnpm build
+node upstream-oracle/run.mjs --pw-repo ~/playwright-upstream
+```
+
+The pin lives in the workflow (`PLAYWRIGHT_UPSTREAM_TAG`) and must move with
+the `@playwright/test` dev dependency; IBR mirrors blob format version 2
+(`BLOB_REPORT_VERSION` in `src/reporter/lifecycle-events.ts`).

package/upstream-oracle/ibr-adapter.js

@@ -0,0 +1,72 @@
+/**
+ * Adapter that lets the IncrementalBlobReporter (IBR) stand in for
+ * Playwright's built-in blob reporter inside the upstream Playwright repo's
+ * test harness (tests/playwright-test).
+ *
+ * Used with PWTEST_INTERMEDIATE_BLOB_REPORT=1 and the fixture patch in
+ * intermediate-mode-ibr.patch: every inline scenario in upstream's runner
+ * suite then executes through IBR -> merge-reports instead of blob ->
+ * merge-reports, turning Playwright's own tests into a completeness oracle
+ * for the IBR blob format. See run.mjs for orchestration.
+ *
+ * A subclass rather than a wrapper so every event handler — including ones
+ * IBR gains later — is inherited automatically instead of hand-forwarded
+ * (a wrapper once silently swallowed onStdOut/onError). The only remaining
+ * bridge is output location: the harness merges <packageRoot>/blob-report/
+ * report-N.zip, IBR writes blob-report-incremental/incremental-report-N.zip,
+ * so the zip is copied over after onEnd. Sharding needs no bridge: IBR falls
+ * back to config.shard when SHARD_INDEX/TOTAL_SHARDS are not set.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const IBR_DIST = process.env.PWTEST_IBR_DIST;
+if (!IBR_DIST) {
+  throw new Error(
+    "PWTEST_IBR_DIST must point at a built incremental-blob-reporter.js " +
+      "(e.g. <monorepo>/packages/playwright-utils/dist/reporter/incremental-blob-reporter.js)",
+  );
+}
+
+// Keep IBR off its local-testing path (attachment embedding + auto-merge)
+// and away from real uploads. Must happen before construction: IBR reads
+// env when instantiated.
+delete process.env.LOCAL_TEST;
+delete process.env.R2_ACCOUNT_ID;
+delete process.env.S3_UPLOAD_BUCKET;
+delete process.env.SHARD_INDEX;
+delete process.env.TOTAL_SHARDS;
+
+const IBR = require(IBR_DIST).default;
+
+class IBRBlobAdapter extends IBR {
+  constructor(options = {}) {
+    super();
+    this._adapterOptions = options;
+    this._adapterCwd = process.cwd();
+  }
+
+  async onEnd(result) {
+    await super.onEnd(result);
+    const shard = this._shardIndex;
+    const src = path.join(
+      this._adapterCwd,
+      "blob-report-incremental",
+      `incremental-report-${shard}.zip`,
+    );
+    if (!fs.existsSync(src)) {
+      // Mirrors blob: no zip when there was nothing to report.
+      return;
+    }
+    const outputDir = path.resolve(
+      this._adapterCwd,
+      this._adapterOptions.outputDir || "blob-report",
+    );
+    const fileName = this._adapterOptions.fileName || `report-${shard}.zip`;
+    fs.mkdirSync(outputDir, { recursive: true });
+    fs.copyFileSync(src, path.join(outputDir, fileName));
+  }
+}
+
+module.exports = IBRBlobAdapter;

package/upstream-oracle/intermediate-mode-ibr.patch

@@ -0,0 +1,16 @@
+diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts
+index 28ef919e2..350fe8b5c 100644
+--- a/tests/playwright-test/playwright-test-fixtures.ts
++++ b/tests/playwright-test/playwright-test-fixtures.ts
+@@ -143,7 +143,10 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
+   let reporter;
+   if (useIntermediateMergeReport) {
+     reporter = params.reporter;
+-    params.reporter = 'blob';
++    // PWTEST_IBR_ADAPTER routes the intermediate report through Empirical's
++    // IncrementalBlobReporter instead of the built-in blob reporter, so this
++    // suite doubles as a blob-format parity oracle for IBR.
++    params.reporter = process.env.PWTEST_IBR_ADAPTER || 'blob';
+   }
+   const reportFile = path.join(baseDir, 'report.json');
+   // When we have useIntermediateMergeReport, we want the JSON reporter only at the merge step.

package/upstream-oracle/run.mjs

@@ -0,0 +1,231 @@
+/**
+ * Runs upstream Playwright's own runner test suite (tests/playwright-test)
+ * through the IncrementalBlobReporter and compares failures against the
+ * built-in blob reporter baseline.
+ *
+ * How: upstream's PWTEST_INTERMEDIATE_BLOB_REPORT=1 mode reroutes every
+ * inline scenario through <reporter> -> merge-reports -> json before applying
+ * the normal assertions — Playwright's own completeness oracle for the blob
+ * format. The fixture patch (intermediate-mode-ibr.patch) lets
+ * PWTEST_IBR_ADAPTER swap IBR in as that reporter.
+ *
+ * Both modes run the same curated spec files; environment noise cancels in
+ * the baseline diff. Any IBR-only failure fails the run — the job stays red
+ * until IBR reaches parity (known gaps today: onStdIO, onError, result-level
+ * annotations, attachment bodies). It turning green is the signal that the
+ * built-in blob reporter is safe to drop.
+ *
+ * Usage:
+ *   node run.mjs --pw-repo <upstream clone at the pinned tag>
+ *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N]
+ *
+ * The clone must be bootstrapped (npm ci + npm run build). This script
+ * (re)applies the fixture patch idempotently on every run.
+ */
+
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ORACLE_DIR = path.dirname(fileURLToPath(import.meta.url));
+
+// Curated, node-only (no browser) runner specs with high report-content
+// signal. stdio.spec.ts is the direct probe for the known onStdIO gap.
+const DEFAULT_SPECS = [
+  "basic.spec.ts",
+  "hooks.spec.ts",
+  "max-failures.spec.ts",
+  "stdio.spec.ts",
+  "test-modifiers.spec.ts",
+  "test-serial.spec.ts",
+  "test-step.spec.ts",
+];
+
+const ADAPTER_PATH = path.join(ORACLE_DIR, "ibr-adapter.js");
+const PATCH_PATH = path.join(ORACLE_DIR, "intermediate-mode-ibr.patch");
+const DEFAULT_IBR_DIST = path.join(
+  ORACLE_DIR,
+  "../dist/reporter/incremental-blob-reporter.js",
+);
+
+function parseArgs(argv) {
+  const args = { specs: DEFAULT_SPECS, workers: 4 };
+  // Normalize --flag=value into --flag value.
+  argv = argv.flatMap((a) =>
+    a.startsWith("--") && a.includes("=")
+      ? [a.slice(0, a.indexOf("=")), a.slice(a.indexOf("=") + 1)]
+      : [a],
+  );
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--pw-repo") {
+      args.pwRepo = argv[++i];
+    } else if (arg === "--specs") {
+      args.specs = argv[++i].split(",").filter(Boolean);
+    } else if (arg === "--workers") {
+      args.workers = Number(argv[++i]);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.pwRepo) {
+    throw new Error(
+      "--pw-repo <path to upstream playwright clone> is required",
+    );
+  }
+  return args;
+}
+
+function run(command, cmdArgs, options) {
+  return new Promise((resolve) => {
+    const child = spawn(command, cmdArgs, {
+      ...options,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let output = "";
+    child.stdout.on("data", (d) => {
+      output += d.toString();
+    });
+    child.stderr.on("data", (d) => {
+      output += d.toString();
+    });
+    child.on("close", (code) => resolve({ code, output }));
+  });
+}
+
+/** Failed-test keys ("file:line › suite › title") from a Playwright JSON report. */
+function collectFailures(report) {
+  const failures = new Set();
+  const visit = (suite, titles) => {
+    for (const spec of suite.specs ?? []) {
+      for (const test of spec.tests ?? []) {
+        if (test.status === "unexpected") {
+          failures.add(
+            [`${spec.file}:${spec.line}`, ...titles, spec.title].join(" › "),
+          );
+        }
+      }
+    }
+    for (const child of suite.suites ?? []) {
+      visit(child, [...titles, child.title]);
+    }
+  };
+  for (const suite of report.suites ?? []) {
+    visit(suite, []);
+  }
+  return failures;
+}
+
+async function applyFixturePatch(pwRepo) {
+  const fixtureFile = "tests/playwright-test/playwright-test-fixtures.ts";
+  const restore = await run("git", ["checkout", "--", fixtureFile], {
+    cwd: pwRepo,
+  });
+  if (restore.code !== 0) {
+    throw new Error(
+      `git checkout of ${fixtureFile} failed:\n${restore.output}`,
+    );
+  }
+  const apply = await run("git", ["apply", PATCH_PATH], { cwd: pwRepo });
+  if (apply.code !== 0) {
+    throw new Error(`git apply ${PATCH_PATH} failed:\n${apply.output}`);
+  }
+}
+
+async function runMode({ pwRepo, specs, workers, mode, ibrDist }) {
+  const jsonOut = path.join(
+    fs.mkdtempSync(path.join(os.tmpdir(), "ibr-oracle-")),
+    `${mode}.json`,
+  );
+  const env = {
+    ...process.env,
+    PWTEST_INTERMEDIATE_BLOB_REPORT: "1",
+    PLAYWRIGHT_JSON_OUTPUT_NAME: jsonOut,
+    FORCE_COLOR: "0",
+  };
+  if (mode === "ibr") {
+    env.PWTEST_IBR_ADAPTER = ADAPTER_PATH;
+    env.PWTEST_IBR_DIST = ibrDist;
+  }
+  const { code, output } = await run(
+    "npm",
+    [
+      "run",
+      "ttest",
+      "--",
+      ...specs,
+      `--workers=${workers}`,
+      // Absorbs outer-harness flake: a scenario that fails on IBR's real gaps
+      // fails every retry, while an infra hiccup passes on retry and is
+      // reported "flaky" (not counted as a failure).
+      "--retries=2",
+      "--reporter=json",
+    ],
+    { cwd: pwRepo, env },
+  );
+  let report;
+  try {
+    report = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+  } catch {
+    console.error(output.slice(-4000));
+    throw new Error(
+      `${mode}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
+    );
+  }
+  return collectFailures(report);
+}
+
+const args = parseArgs(process.argv.slice(2));
+const pwRepo = path.resolve(args.pwRepo);
+const ibrDist = path.resolve(process.env.PWTEST_IBR_DIST || DEFAULT_IBR_DIST);
+if (!fs.existsSync(ibrDist)) {
+  throw new Error(
+    `IBR dist not found at ${ibrDist} — build playwright-utils first`,
+  );
+}
+
+await applyFixturePatch(pwRepo);
+
+console.log(`Specs: ${args.specs.join(", ")}`);
+console.log("Running baseline (built-in blob reporter)...");
+const blobFailures = await runMode({
+  pwRepo,
+  specs: args.specs,
+  workers: args.workers,
+  mode: "blob",
+});
+console.log(
+  `Baseline failures (env noise, cancelled in diff): ${blobFailures.size}`,
+);
+for (const key of [...blobFailures].sort()) {
+  console.log(`  [baseline] ${key}`);
+}
+
+console.log("Running through IBR adapter...");
+const ibrFailures = await runMode({
+  pwRepo,
+  specs: args.specs,
+  workers: args.workers,
+  mode: "ibr",
+  ibrDist,
+});
+
+const ibrOnly = [...ibrFailures].filter((key) => !blobFailures.has(key)).sort();
+console.log(`IBR-only failures: ${ibrOnly.length}`);
+for (const key of ibrOnly) {
+  console.log(`  [ibr-only] ${key}`);
+}
+
+if (ibrOnly.length > 0) {
+  console.error(
+    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR. ` +
+      `The job stays red until IBR reaches blob parity (known gaps: onStdIO, ` +
+      `onError, result-level annotations, attachment bodies).`,
+  );
+  process.exit(1);
+}
+console.log(
+  "\nOK: IBR at parity with the built-in blob reporter on these specs",
+);