@empiricalrun/playwright-utils 0.58.2 → 0.58.4

package/dist/reporter/parity-utils.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeJsonReport = normalizeJsonReport;
+exports.stepSignatures = stepSignatures;
+exports.missingEventMethods = missingEventMethods;
+const ANSI_RE = /\u001b\[[0-9;]*m/g;
+function normalizeText(text, cwd) {
+    return text.replace(ANSI_RE, "").split(cwd).join("<cwd>").trimEnd();
+}
+function normalizeStdio(entries, cwd) {
+    return (entries ?? []).map((entry) => typeof entry === "string"
+        ? normalizeText(entry, cwd)
+        : entry.text !== undefined
+            ? normalizeText(entry.text, cwd)
+            : `<buffer:${entry.buffer?.length ?? 0}>`);
+}
+function normalizeResult(result, opts) {
+    return {
+        retry: result.retry,
+        status: result.status,
+        errorMessages: (result.errors ?? []).map((e) => normalizeText(e.message ?? "", opts.cwd)),
+        stdout: normalizeStdio(result.stdout, opts.cwd),
+        stderr: normalizeStdio(result.stderr, opts.cwd),
+        attachments: (result.attachments ?? []).map((a) => ({
+            name: a.name,
+            contentType: a.contentType,
+            hasContent: opts.compareAttachmentContent
+                ? Boolean(a.path || a.body)
+                : true,
+        })),
+    };
+}
+function normalizeSuite(suite, opts) {
+    return {
+        title: suite.title,
+        file: suite.file,
+        specs: (suite.specs ?? [])
+            .map((spec) => ({
+            title: spec.title,
+            file: spec.file,
+            ok: spec.ok,
+            tests: (spec.tests ?? []).map((test) => ({
+                projectName: test.projectName,
+                expectedStatus: test.expectedStatus,
+                status: test.status,
+                annotations: test.annotations ?? [],
+                results: (test.results ?? [])
+                    .map((r) => normalizeResult(r, opts))
+                    .sort((a, b) => a.retry - b.retry),
+            })),
+        }))
+            .sort((a, b) => a.title.localeCompare(b.title)),
+        suites: (suite.suites ?? [])
+            .map((s) => normalizeSuite(s, opts))
+            .sort((a, b) => a.title.localeCompare(b.title)),
+    };
+}
+function normalizeJsonReport(report, opts) {
+    const typed = report;
+    const stats = { ...(typed.stats ?? {}) };
+    delete stats.startTime;
+    delete stats.duration;
+    return {
+        suites: (typed.suites ?? [])
+            .map((s) => normalizeSuite(s, opts))
+            .sort((a, b) => (a.file + a.title).localeCompare(b.file + b.title)),
+        errorMessages: (typed.errors ?? []).map((e) => normalizeText(e.message ?? "", opts.cwd)),
+        stats,
+    };
+}
+/**
+ * Per-result step signatures from a raw blob event stream, keyed by
+ * `testId:retry` (stable across both reporters — they see the same run).
+ * The merged JSON report drops steps, so step parity is checked here.
+ */
+function stepSignatures(events) {
+    const resultKey = new Map(); // resultId -> testId:retry
+    for (const event of events) {
+        if (event.method !== "onTestBegin") {
+            continue;
+        }
+        const params = event.params;
+        resultKey.set(params.result.id, `${params.testId}:${params.result.retry}`);
+    }
+    const signatures = new Map();
+    for (const event of events) {
+        if (event.method !== "onStepBegin") {
+            continue;
+        }
+        const params = event.params;
+        const key = resultKey.get(params.resultId);
+        if (!key) {
+            continue;
+        }
+        const list = signatures.get(key) ?? [];
+        list.push(`${params.step.category}:${params.step.title}`);
+        signatures.set(key, list);
+    }
+    return signatures;
+}
+/** Event method names present in `reference` but absent from `candidate`. */
+function missingEventMethods(reference, candidate) {
+    const candidateMethods = new Set(candidate.map((e) => e.method));
+    return [
+        ...new Set(reference
+            .map((e) => e.method)
+            .filter((method) => !candidateMethods.has(method))),
+    ].sort();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.2",
+  "version": "0.58.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "openapi-typescript": "7.6.1",
     "playwright-core": "1.62.1",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.15.3"
+    "@empiricalrun/shared-types": "0.15.4"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}
+{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/parity-utils.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}

package/upstream-oracle/README.md

@@ -0,0 +1,48 @@
+# Upstream Playwright oracle for the IncrementalBlobReporter
+
+Runs upstream Playwright's own runner test suite (`tests/playwright-test`)
+through our IncrementalBlobReporter (IBR) and diffs failures against the
+built-in blob reporter as baseline. Their
+`PWTEST_INTERMEDIATE_BLOB_REPORT=1` harness mode reroutes every inline
+scenario through `<reporter> -> merge-reports -> json` before applying the
+normal assertions — Playwright's own completeness check for the blob format.
+Swapping IBR in (via `ibr-adapter.js` + the one-line
+`intermediate-mode-ibr.patch`) turns it into a parity oracle for us.
+
+This backs the plan to drop the built-in blob reporter from sharded runs:
+before trusting IBR as the only report, its zip must merge identically on
+scenarios we did not think to write ourselves. The fast, per-PR half of that
+story is `src/reporter/blob-ibr-parity.test.ts`; this oracle is the deep
+audit, run by `.github/workflows/ibr-blob-oracle.yaml` (and re-run on every
+Playwright version-pin bump).
+
+## Files
+
+- `run.mjs` — orchestrator: applies the patch, runs the curated specs in both
+  modes, fails on any IBR-only failure.
+- `ibr-adapter.js` — bridges IBR's env/paths contract to the blob reporter's
+  (config.shard → SHARD_INDEX env, zip relocation to `blob-report/`).
+- `intermediate-mode-ibr.patch` — one-line upstream fixture change making the
+  intermediate reporter selectable via `PWTEST_IBR_ADAPTER`.
+
+The job is EXPECTED to stay red until IBR reaches parity: today ~55 upstream
+tests fail only through IBR, all tracing to the known gaps (`onStdIO`,
+`onError`, result-level annotations, attachment bodies — encoded as `it.fails`
+in `src/reporter/blob-ibr-parity.test.ts`). The failure count shrinking to
+zero is the signal that the built-in blob reporter is safe to drop.
+
+## Running locally
+
+```bash
+# One-time: clone + bootstrap upstream at the pinned tag (~10 min)
+git clone --depth 1 --branch v1.62.1 https://github.com/microsoft/playwright.git ~/playwright-upstream
+cd ~/playwright-upstream && npm ci && npm run build
+
+# Build IBR, then run the oracle
+cd <monorepo>/packages/playwright-utils && pnpm build
+node upstream-oracle/run.mjs --pw-repo ~/playwright-upstream
+```
+
+The pin lives in the workflow (`PLAYWRIGHT_UPSTREAM_TAG`) and must move with
+the `@playwright/test` dev dependency; IBR mirrors blob format version 2
+(`BLOB_REPORT_VERSION` in `src/reporter/lifecycle-events.ts`).

package/upstream-oracle/ibr-adapter.js

@@ -0,0 +1,72 @@
+/**
+ * Adapter that lets the IncrementalBlobReporter (IBR) stand in for
+ * Playwright's built-in blob reporter inside the upstream Playwright repo's
+ * test harness (tests/playwright-test).
+ *
+ * Used with PWTEST_INTERMEDIATE_BLOB_REPORT=1 and the fixture patch in
+ * intermediate-mode-ibr.patch: every inline scenario in upstream's runner
+ * suite then executes through IBR -> merge-reports instead of blob ->
+ * merge-reports, turning Playwright's own tests into a completeness oracle
+ * for the IBR blob format. See run.mjs for orchestration.
+ *
+ * A subclass rather than a wrapper so every event handler — including ones
+ * IBR gains later — is inherited automatically instead of hand-forwarded
+ * (a wrapper once silently swallowed onStdOut/onError). The only remaining
+ * bridge is output location: the harness merges <packageRoot>/blob-report/
+ * report-N.zip, IBR writes blob-report-incremental/incremental-report-N.zip,
+ * so the zip is copied over after onEnd. Sharding needs no bridge: IBR falls
+ * back to config.shard when SHARD_INDEX/TOTAL_SHARDS are not set.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const IBR_DIST = process.env.PWTEST_IBR_DIST;
+if (!IBR_DIST) {
+  throw new Error(
+    "PWTEST_IBR_DIST must point at a built incremental-blob-reporter.js " +
+      "(e.g. <monorepo>/packages/playwright-utils/dist/reporter/incremental-blob-reporter.js)",
+  );
+}
+
+// Keep IBR off its local-testing path (attachment embedding + auto-merge)
+// and away from real uploads. Must happen before construction: IBR reads
+// env when instantiated.
+delete process.env.LOCAL_TEST;
+delete process.env.R2_ACCOUNT_ID;
+delete process.env.S3_UPLOAD_BUCKET;
+delete process.env.SHARD_INDEX;
+delete process.env.TOTAL_SHARDS;
+
+const IBR = require(IBR_DIST).default;
+
+class IBRBlobAdapter extends IBR {
+  constructor(options = {}) {
+    super();
+    this._adapterOptions = options;
+    this._adapterCwd = process.cwd();
+  }
+
+  async onEnd(result) {
+    await super.onEnd(result);
+    const shard = this._shardIndex;
+    const src = path.join(
+      this._adapterCwd,
+      "blob-report-incremental",
+      `incremental-report-${shard}.zip`,
+    );
+    if (!fs.existsSync(src)) {
+      // Mirrors blob: no zip when there was nothing to report.
+      return;
+    }
+    const outputDir = path.resolve(
+      this._adapterCwd,
+      this._adapterOptions.outputDir || "blob-report",
+    );
+    const fileName = this._adapterOptions.fileName || `report-${shard}.zip`;
+    fs.mkdirSync(outputDir, { recursive: true });
+    fs.copyFileSync(src, path.join(outputDir, fileName));
+  }
+}
+
+module.exports = IBRBlobAdapter;

package/upstream-oracle/intermediate-mode-ibr.patch

@@ -0,0 +1,16 @@
+diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts
+index 28ef919e2..350fe8b5c 100644
+--- a/tests/playwright-test/playwright-test-fixtures.ts
++++ b/tests/playwright-test/playwright-test-fixtures.ts
+@@ -143,7 +143,10 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
+   let reporter;
+   if (useIntermediateMergeReport) {
+     reporter = params.reporter;
+-    params.reporter = 'blob';
++    // PWTEST_IBR_ADAPTER routes the intermediate report through Empirical's
++    // IncrementalBlobReporter instead of the built-in blob reporter, so this
++    // suite doubles as a blob-format parity oracle for IBR.
++    params.reporter = process.env.PWTEST_IBR_ADAPTER || 'blob';
+   }
+   const reportFile = path.join(baseDir, 'report.json');
+   // When we have useIntermediateMergeReport, we want the JSON reporter only at the merge step.

package/upstream-oracle/run.mjs

@@ -0,0 +1,395 @@
+/**
+ * Runs upstream Playwright's own runner test suite (tests/playwright-test)
+ * through the IncrementalBlobReporter and compares failures against the
+ * built-in blob reporter baseline.
+ *
+ * How: upstream's PWTEST_INTERMEDIATE_BLOB_REPORT=1 mode reroutes every
+ * inline scenario through <reporter> -> merge-reports -> json before applying
+ * the normal assertions — Playwright's own completeness oracle for the blob
+ * format. The fixture patch (intermediate-mode-ibr.patch) lets
+ * PWTEST_IBR_ADAPTER swap IBR in as that reporter.
+ *
+ * Both modes run the same spec files; environment noise cancels in the
+ * baseline diff. Any IBR-only failure fails the run — the job stays red
+ * until IBR reaches parity on these specs. It staying green is the signal
+ * that the built-in blob reporter is safe to drop.
+ *
+ * Usage:
+ *   node run.mjs --pw-repo <upstream clone at the pinned tag>
+ *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N] [--shard i/n]
+ *
+ * The clone must be bootstrapped (npm ci + npm run build). This script
+ * (re)applies the fixture patch idempotently on every run.
+ */
+
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ORACLE_DIR = path.dirname(fileURLToPath(import.meta.url));
+
+// The default sweep is EVERY spec in upstream's tests/playwright-test —
+// IBR is meant to become the default blob reporter, so it has to stand the
+// whole runner suite. Only structurally out-of-scope specs are excluded,
+// each with its reason printed at startup (no silent caps). First matching
+// rule wins.
+const EXCLUSION_RULES = [
+  [
+    (name) => name.startsWith("playwright.ct-"),
+    "component-testing build; no runner-report signal",
+  ],
+  [
+    (name) => name.startsWith("playwright."),
+    "drives real browsers (not installed in the oracle env)",
+  ],
+  [
+    (name) => name.startsWith("ui-mode-"),
+    "UI mode; needs trace viewer + browsers",
+  ],
+  [
+    (name) =>
+      name === "to-have-screenshot.spec.ts" ||
+      name === "aria-snapshot-file.spec.ts" ||
+      name === "update-aria-snapshot.spec.ts",
+    "browser screenshots/aria snapshots",
+  ],
+  [
+    (name) => name === "types.spec.ts" || name === "types-2.spec.ts",
+    "tsc-only; exercises no runner report",
+  ],
+  [
+    (name) => name === "reporter-html.spec.ts",
+    "browser-driven HTML-report UI tests; no blob-format signal (merge " +
+      "semantics are covered by reporter-blob.spec.ts) and they busy-loop " +
+      "workers under load, wedging the sweep",
+  ],
+];
+
+// Tests skipped by title via --grep-invert. Reserved for tests that WEDGE
+// the sweep — a busy-looped worker no per-test timeout can interrupt — so
+// exclusion is surgical (exact title match) and every entry carries the
+// observed reason. Everything else in the spec still runs.
+const HANGING_TESTS = [];
+
+function discoverSpecs(pwRepo) {
+  const specDir = path.join(pwRepo, "tests", "playwright-test");
+  const all = fs
+    .readdirSync(specDir)
+    .filter((f) => f.endsWith(".spec.ts"))
+    .sort();
+  const included = [];
+  const excluded = [];
+  for (const name of all) {
+    const rule = EXCLUSION_RULES.find(([match]) => match(name));
+    if (rule) {
+      excluded.push([name, rule[1]]);
+    } else {
+      included.push(name);
+    }
+  }
+  return { included, excluded };
+}
+
+const ADAPTER_PATH = path.join(ORACLE_DIR, "ibr-adapter.js");
+const PATCH_PATH = path.join(ORACLE_DIR, "intermediate-mode-ibr.patch");
+const DEFAULT_IBR_DIST = path.join(
+  ORACLE_DIR,
+  "../dist/reporter/incremental-blob-reporter.js",
+);
+
+function parseArgs(argv) {
+  const args = { specs: null, workers: 4, shard: null };
+  // Normalize --flag=value into --flag value.
+  argv = argv.flatMap((a) =>
+    a.startsWith("--") && a.includes("=")
+      ? [a.slice(0, a.indexOf("=")), a.slice(a.indexOf("=") + 1)]
+      : [a],
+  );
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--pw-repo") {
+      args.pwRepo = argv[++i];
+    } else if (arg === "--specs") {
+      args.specs = argv[++i].split(",").filter(Boolean);
+    } else if (arg === "--workers") {
+      args.workers = Number(argv[++i]);
+    } else if (arg === "--shard") {
+      const match = /^(\d+)\/(\d+)$/.exec(argv[++i] ?? "");
+      if (!match) {
+        throw new Error("--shard expects i/n (e.g. --shard 2/4)");
+      }
+      args.shard = { index: Number(match[1]), total: Number(match[2]) };
+      if (args.shard.index < 1 || args.shard.index > args.shard.total) {
+        throw new Error(`--shard index out of range: ${argv[i]}`);
+      }
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.pwRepo) {
+    throw new Error(
+      "--pw-repo <path to upstream playwright clone> is required",
+    );
+  }
+  return args;
+}
+
+function run(command, cmdArgs, options) {
+  return new Promise((resolve) => {
+    const child = spawn(command, cmdArgs, {
+      ...options,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let output = "";
+    child.stdout.on("data", (d) => {
+      output += d.toString();
+    });
+    child.stderr.on("data", (d) => {
+      output += d.toString();
+    });
+    child.on("close", (code) => resolve({ code, output }));
+  });
+}
+
+/** Failed-test keys ("file:line › suite › title") from a Playwright JSON report. */
+function collectFailures(report) {
+  const failures = new Set();
+  const visit = (suite, titles) => {
+    for (const spec of suite.specs ?? []) {
+      for (const test of spec.tests ?? []) {
+        if (test.status === "unexpected") {
+          failures.add(
+            [`${spec.file}:${spec.line}`, ...titles, spec.title].join(" › "),
+          );
+        }
+      }
+    }
+    for (const child of suite.suites ?? []) {
+      visit(child, [...titles, child.title]);
+    }
+  };
+  for (const suite of report.suites ?? []) {
+    visit(suite, []);
+  }
+  return failures;
+}
+
+async function applyFixturePatch(pwRepo) {
+  const fixtureFile = "tests/playwright-test/playwright-test-fixtures.ts";
+  const restore = await run("git", ["checkout", "--", fixtureFile], {
+    cwd: pwRepo,
+  });
+  if (restore.code !== 0) {
+    throw new Error(
+      `git checkout of ${fixtureFile} failed:\n${restore.output}`,
+    );
+  }
+  const apply = await run("git", ["apply", PATCH_PATH], { cwd: pwRepo });
+  if (apply.code !== 0) {
+    throw new Error(`git apply ${PATCH_PATH} failed:\n${apply.output}`);
+  }
+}
+
+// Hard cap per spec-mode run (milliseconds — Playwright's --global-timeout
+// unit). Some upstream tests can busy-loop a worker under load (sync string
+// processing that no per-test timeout can interrupt — the timeouts run on
+// the same blocked event loop); which test wedges is timing-dependent, so
+// title exclusion cannot converge. Playwright's global timeout fires from
+// the main process and SIGKILLs through busy loops. Each spec runs as its
+// own bounded invocation, so a wedge costs one spec's cap, names the spec,
+// and the sweep continues — hangs become inventory instead of aborting the
+// whole mode.
+const SPEC_TIMEOUT_MS = 10 * 60 * 1_000;
+
+async function runMode({ pwRepo, spec, workers, mode, ibrDist }) {
+  const startedAt = Date.now();
+  const jsonOut = path.join(
+    fs.mkdtempSync(path.join(os.tmpdir(), "ibr-oracle-")),
+    `${mode}.json`,
+  );
+  const env = {
+    ...process.env,
+    PWTEST_INTERMEDIATE_BLOB_REPORT: "1",
+    PLAYWRIGHT_JSON_OUTPUT_NAME: jsonOut,
+    FORCE_COLOR: "0",
+  };
+  if (mode === "ibr") {
+    env.PWTEST_IBR_ADAPTER = ADAPTER_PATH;
+    env.PWTEST_IBR_DIST = ibrDist;
+  }
+  // The inner run writes its JSON report at the end, so nothing streams to
+  // stdout while it runs. Heartbeat from the test-results dir (the outer
+  // harness creates one dir per executed test) so long modes show progress.
+  const testResultsDir = path.join(pwRepo, "test-results");
+  const heartbeat = setInterval(() => {
+    let dirs = 0;
+    try {
+      dirs = fs.readdirSync(testResultsDir).length;
+    } catch {
+      // Not created yet.
+    }
+    const minutes = Math.round((Date.now() - startedAt) / 60_000);
+    console.log(
+      `  [${mode} ${spec}] ~${dirs} tests executed, ${minutes}m elapsed`,
+    );
+  }, 60_000);
+  const { code, output } = await run(
+    "npm",
+    [
+      "run",
+      "ttest",
+      "--",
+      spec,
+      `--workers=${workers}`,
+      // Absorbs outer-harness flake: a scenario that fails on IBR's real gaps
+      // fails every retry, while an infra hiccup passes on retry and is
+      // reported "flaky" (not counted as a failure).
+      "--retries=2",
+      `--global-timeout=${SPEC_TIMEOUT_MS}`,
+      // An empty --grep-invert would match (and thus drop) every test, so
+      // the flag only appears when there are titles to skip.
+      ...(HANGING_TESTS.length
+        ? [`--grep-invert=${HANGING_TESTS.map((h) => h.pattern).join("|")}`]
+        : []),
+      "--reporter=json",
+    ],
+    { cwd: pwRepo, env },
+  );
+  clearInterval(heartbeat);
+  const elapsedSeconds = Math.round((Date.now() - startedAt) / 1_000);
+  // A globally-timed-out run still writes a valid JSON report with the
+  // remaining tests marked "did not run", which would otherwise diff as a
+  // false green — report the hang instead of trusting the partial report.
+  if (/Timed out waiting .* for the test suite/.test(output)) {
+    return { hung: true, elapsedSeconds };
+  }
+  let report;
+  try {
+    report = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
+  } catch {
+    console.error(output.slice(-4000));
+    throw new Error(
+      `${mode} ${spec}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
+    );
+  }
+  const stats = report.stats ?? {};
+  const totalRun =
+    (stats.expected ?? 0) +
+    (stats.unexpected ?? 0) +
+    (stats.flaky ?? 0) +
+    (stats.skipped ?? 0);
+  return { failures: collectFailures(report), totalRun, elapsedSeconds };
+}
+
+const args = parseArgs(process.argv.slice(2));
+const pwRepo = path.resolve(args.pwRepo);
+const ibrDist = path.resolve(process.env.PWTEST_IBR_DIST || DEFAULT_IBR_DIST);
+if (!fs.existsSync(ibrDist)) {
+  throw new Error(
+    `IBR dist not found at ${ibrDist} — build playwright-utils first`,
+  );
+}
+
+await applyFixturePatch(pwRepo);
+
+if (!args.specs) {
+  const { included, excluded } = discoverSpecs(pwRepo);
+  args.specs = included;
+  console.log(`Excluded specs (${excluded.length}):`);
+  for (const [name, reason] of excluded) {
+    console.log(`  [excluded] ${name} — ${reason}`);
+  }
+}
+if (args.shard) {
+  // Round-robin slice so slow specs spread across shards.
+  args.specs = args.specs.filter(
+    (_, i) => i % args.shard.total === args.shard.index - 1,
+  );
+  console.log(
+    `Shard ${args.shard.index}/${args.shard.total}: ${args.specs.length} specs`,
+  );
+}
+
+console.log(`Specs: ${args.specs.join(", ")}`);
+for (const { pattern, reason } of HANGING_TESTS) {
+  console.log(`  [skipped-by-title] /${pattern}/ — ${reason}`);
+}
+// Each spec runs both modes back to back as its own bounded invocation
+// (see SPEC_TIMEOUT_MS); a hang or count mismatch quarantines that spec and
+// the sweep continues.
+const ibrOnly = [];
+const hungSpecs = [];
+const mismatchedSpecs = [];
+let baselineNoise = 0;
+
+for (const spec of args.specs) {
+  const blob = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "blob",
+  });
+  if (blob.hung) {
+    console.log(`  [hung] ${spec} (blob mode, ${blob.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (blob mode)`);
+    continue;
+  }
+  const ibr = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "ibr",
+    ibrDist,
+  });
+  if (ibr.hung) {
+    console.log(`  [hung] ${spec} (ibr mode, ${ibr.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (ibr mode)`);
+    continue;
+  }
+  if (blob.totalRun === 0 || blob.totalRun !== ibr.totalRun) {
+    console.log(
+      `  [mismatch] ${spec}: blob ran ${blob.totalRun}, ibr ran ${ibr.totalRun}`,
+    );
+    mismatchedSpecs.push(spec);
+    continue;
+  }
+  const specIbrOnly = [...ibr.failures]
+    .filter((key) => !blob.failures.has(key))
+    .sort();
+  baselineNoise += blob.failures.size;
+  ibrOnly.push(...specIbrOnly);
+  console.log(
+    `  ${spec}: ${blob.totalRun} tests, ` +
+      `${blob.failures.size} baseline noise, ${specIbrOnly.length} ibr-only ` +
+      `(blob ${blob.elapsedSeconds}s, ibr ${ibr.elapsedSeconds}s)`,
+  );
+}
+
+console.log(
+  `\nBaseline failures (env noise, cancelled in diff): ${baselineNoise}`,
+);
+console.log(`IBR-only failures: ${ibrOnly.length}`);
+for (const key of ibrOnly.sort()) {
+  console.log(`  [ibr-only] ${key}`);
+}
+for (const spec of hungSpecs) {
+  console.log(`  [hung] ${spec}`);
+}
+for (const spec of mismatchedSpecs) {
+  console.log(`  [mismatch] ${spec}`);
+}
+
+if (ibrOnly.length > 0 || hungSpecs.length > 0 || mismatchedSpecs.length > 0) {
+  console.error(
+    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR, ` +
+      `${hungSpecs.length} spec run(s) hung, ${mismatchedSpecs.length} ` +
+      `spec(s) mismatched. The job stays red until IBR reaches blob parity ` +
+      `on these specs.`,
+  );
+  process.exit(1);
+}
+console.log(
+  "\nOK: IBR at parity with the built-in blob reporter on these specs",
+);