@empiricalrun/playwright-utils 0.58.3 → 0.58.4

package/dist/reporter/empirical-reporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"empirical-reporter.d.ts","sourceRoot":"","sources":["../../src/reporter/empirical-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,UAAU,EACX,MAAM,2BAA2B,CAAC;AAuBnC,cAAM,iBAAkB,YAAW,QAAQ;IACzC,OAAO,CAAC,kBAAkB,CAAyB;IACnD,OAAO,CAAC,oBAAoB,CAG1B;IACF,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,YAAY,CAAgD;IACpE,OAAO,CAAC,iBAAiB,CAAyC;IAClE,OAAO,CAAC,gBAAgB,CAAkC;IAC1D,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,kBAAkB,CAA2B;IAGrD,OAAO,CAAC,YAAY,CAAqB;IAEzC,cAEC;IAED,OAAO,CAAC,MAAM,EAAE,UAAU,QAEzB;IAED,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,6BAA6B,CA2CnC;IAEF,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,QAqI3C;IAEK,KAAK,CAAC,MAAM,EAAE,UAAU,iBAiH7B;IAED,OAAO,CAAC,qBAAqB;YAkBf,gBAAgB;YAOhB,iBAAiB;CAqChC;eAEc,iBAAiB"}
+{"version":3,"file":"empirical-reporter.d.ts","sourceRoot":"","sources":["../../src/reporter/empirical-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,UAAU,EACX,MAAM,2BAA2B,CAAC;AAuBnC,cAAM,iBAAkB,YAAW,QAAQ;IACzC,OAAO,CAAC,kBAAkB,CAAyB;IACnD,OAAO,CAAC,oBAAoB,CAG1B;IACF,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,YAAY,CAAgD;IACpE,OAAO,CAAC,iBAAiB,CAAyC;IAClE,OAAO,CAAC,gBAAgB,CAAkC;IAC1D,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,kBAAkB,CAA2B;IAGrD,OAAO,CAAC,YAAY,CAAqB;IAEzC,cAEC;IAED,OAAO,CAAC,MAAM,EAAE,UAAU,QAEzB;IAED,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,6BAA6B,CA2CnC;IAEF,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,QA+I3C;IAEK,KAAK,CAAC,MAAM,EAAE,UAAU,iBAiH7B;IAED,OAAO,CAAC,qBAAqB;YAkBf,gBAAgB;YAOhB,iBAAiB;CAqChC;eAEc,iBAAiB"}

package/dist/reporter/empirical-reporter.js

@@ -150,6 +150,15 @@ class EmpiricalReporter {
                                 stack: e.stack,
                                 location: e.location,
                             })),
+                            // Playwright seeds result.annotations with the full per-attempt
+                            // set (static + suite + runtime-pushed); test.annotations is the
+                            // fallback for older Playwright versions without it.
+                            annotations: (result.annotations?.length
+                                ? result.annotations
+                                : test.annotations).map((a) => ({
+                                type: a.type,
+                                description: a.description,
+                            })),
                         },
                         projectName,
                         attachments: successfulAttachments,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.3",
+  "version": "0.58.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "openapi-typescript": "7.6.1",
     "playwright-core": "1.62.1",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.15.3"
+    "@empiricalrun/shared-types": "0.15.4"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
@@ -54,8 +54,8 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/llm": "^0.27.0",
     "@empiricalrun/cua": "^0.4.1",
+    "@empiricalrun/llm": "^0.27.0",
     "@empiricalrun/r2-uploader": "^0.10.1",
     "@empiricalrun/reporter": "^0.29.1"
   },

package/upstream-oracle/run.mjs

@@ -9,15 +9,14 @@
  * format. The fixture patch (intermediate-mode-ibr.patch) lets
  * PWTEST_IBR_ADAPTER swap IBR in as that reporter.
  *
- * Both modes run the same curated spec files; environment noise cancels in
- * the baseline diff. Any IBR-only failure fails the run — the job stays red
- * until IBR reaches parity (known gaps today: onStdIO, onError, result-level
- * annotations, attachment bodies). It turning green is the signal that the
- * built-in blob reporter is safe to drop.
+ * Both modes run the same spec files; environment noise cancels in the
+ * baseline diff. Any IBR-only failure fails the run — the job stays red
+ * until IBR reaches parity on these specs. It staying green is the signal
+ * that the built-in blob reporter is safe to drop.
  *
  * Usage:
  *   node run.mjs --pw-repo <upstream clone at the pinned tag>
- *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N]
+ *     [--specs basic.spec.ts,stdio.spec.ts] [--workers N] [--shard i/n]
  *
  * The clone must be bootstrapped (npm ci + npm run build). This script
  * (re)applies the fixture patch idempotently on every run.
@@ -31,18 +30,68 @@ import { fileURLToPath } from "node:url";
 
 const ORACLE_DIR = path.dirname(fileURLToPath(import.meta.url));
 
-// Curated, node-only (no browser) runner specs with high report-content
-// signal. stdio.spec.ts is the direct probe for the known onStdIO gap.
-const DEFAULT_SPECS = [
-  "basic.spec.ts",
-  "hooks.spec.ts",
-  "max-failures.spec.ts",
-  "stdio.spec.ts",
-  "test-modifiers.spec.ts",
-  "test-serial.spec.ts",
-  "test-step.spec.ts",
+// The default sweep is EVERY spec in upstream's tests/playwright-test —
+// IBR is meant to become the default blob reporter, so it has to stand the
+// whole runner suite. Only structurally out-of-scope specs are excluded,
+// each with its reason printed at startup (no silent caps). First matching
+// rule wins.
+const EXCLUSION_RULES = [
+  [
+    (name) => name.startsWith("playwright.ct-"),
+    "component-testing build; no runner-report signal",
+  ],
+  [
+    (name) => name.startsWith("playwright."),
+    "drives real browsers (not installed in the oracle env)",
+  ],
+  [
+    (name) => name.startsWith("ui-mode-"),
+    "UI mode; needs trace viewer + browsers",
+  ],
+  [
+    (name) =>
+      name === "to-have-screenshot.spec.ts" ||
+      name === "aria-snapshot-file.spec.ts" ||
+      name === "update-aria-snapshot.spec.ts",
+    "browser screenshots/aria snapshots",
+  ],
+  [
+    (name) => name === "types.spec.ts" || name === "types-2.spec.ts",
+    "tsc-only; exercises no runner report",
+  ],
+  [
+    (name) => name === "reporter-html.spec.ts",
+    "browser-driven HTML-report UI tests; no blob-format signal (merge " +
+      "semantics are covered by reporter-blob.spec.ts) and they busy-loop " +
+      "workers under load, wedging the sweep",
+  ],
 ];
 
+// Tests skipped by title via --grep-invert. Reserved for tests that WEDGE
+// the sweep — a busy-looped worker no per-test timeout can interrupt — so
+// exclusion is surgical (exact title match) and every entry carries the
+// observed reason. Everything else in the spec still runs.
+const HANGING_TESTS = [];
+
+function discoverSpecs(pwRepo) {
+  const specDir = path.join(pwRepo, "tests", "playwright-test");
+  const all = fs
+    .readdirSync(specDir)
+    .filter((f) => f.endsWith(".spec.ts"))
+    .sort();
+  const included = [];
+  const excluded = [];
+  for (const name of all) {
+    const rule = EXCLUSION_RULES.find(([match]) => match(name));
+    if (rule) {
+      excluded.push([name, rule[1]]);
+    } else {
+      included.push(name);
+    }
+  }
+  return { included, excluded };
+}
+
 const ADAPTER_PATH = path.join(ORACLE_DIR, "ibr-adapter.js");
 const PATCH_PATH = path.join(ORACLE_DIR, "intermediate-mode-ibr.patch");
 const DEFAULT_IBR_DIST = path.join(
@@ -51,7 +100,7 @@ const DEFAULT_IBR_DIST = path.join(
 );
 
 function parseArgs(argv) {
-  const args = { specs: DEFAULT_SPECS, workers: 4 };
+  const args = { specs: null, workers: 4, shard: null };
   // Normalize --flag=value into --flag value.
   argv = argv.flatMap((a) =>
     a.startsWith("--") && a.includes("=")
@@ -66,6 +115,15 @@ function parseArgs(argv) {
       args.specs = argv[++i].split(",").filter(Boolean);
     } else if (arg === "--workers") {
       args.workers = Number(argv[++i]);
+    } else if (arg === "--shard") {
+      const match = /^(\d+)\/(\d+)$/.exec(argv[++i] ?? "");
+      if (!match) {
+        throw new Error("--shard expects i/n (e.g. --shard 2/4)");
+      }
+      args.shard = { index: Number(match[1]), total: Number(match[2]) };
+      if (args.shard.index < 1 || args.shard.index > args.shard.total) {
+        throw new Error(`--shard index out of range: ${argv[i]}`);
+      }
     } else {
       throw new Error(`Unknown argument: ${arg}`);
     }
@@ -134,7 +192,19 @@ async function applyFixturePatch(pwRepo) {
   }
 }
 
-async function runMode({ pwRepo, specs, workers, mode, ibrDist }) {
+// Hard cap per spec-mode run (milliseconds — Playwright's --global-timeout
+// unit). Some upstream tests can busy-loop a worker under load (sync string
+// processing that no per-test timeout can interrupt — the timeouts run on
+// the same blocked event loop); which test wedges is timing-dependent, so
+// title exclusion cannot converge. Playwright's global timeout fires from
+// the main process and SIGKILLs through busy loops. Each spec runs as its
+// own bounded invocation, so a wedge costs one spec's cap, names the spec,
+// and the sweep continues — hangs become inventory instead of aborting the
+// whole mode.
+const SPEC_TIMEOUT_MS = 10 * 60 * 1_000;
+
+async function runMode({ pwRepo, spec, workers, mode, ibrDist }) {
+  const startedAt = Date.now();
   const jsonOut = path.join(
     fs.mkdtempSync(path.join(os.tmpdir(), "ibr-oracle-")),
     `${mode}.json`,
@@ -149,32 +219,68 @@ async function runMode({ pwRepo, specs, workers, mode, ibrDist }) {
     env.PWTEST_IBR_ADAPTER = ADAPTER_PATH;
     env.PWTEST_IBR_DIST = ibrDist;
   }
+  // The inner run writes its JSON report at the end, so nothing streams to
+  // stdout while it runs. Heartbeat from the test-results dir (the outer
+  // harness creates one dir per executed test) so long modes show progress.
+  const testResultsDir = path.join(pwRepo, "test-results");
+  const heartbeat = setInterval(() => {
+    let dirs = 0;
+    try {
+      dirs = fs.readdirSync(testResultsDir).length;
+    } catch {
+      // Not created yet.
+    }
+    const minutes = Math.round((Date.now() - startedAt) / 60_000);
+    console.log(
+      `  [${mode} ${spec}] ~${dirs} tests executed, ${minutes}m elapsed`,
+    );
+  }, 60_000);
   const { code, output } = await run(
     "npm",
     [
       "run",
       "ttest",
       "--",
-      ...specs,
+      spec,
       `--workers=${workers}`,
       // Absorbs outer-harness flake: a scenario that fails on IBR's real gaps
       // fails every retry, while an infra hiccup passes on retry and is
       // reported "flaky" (not counted as a failure).
       "--retries=2",
+      `--global-timeout=${SPEC_TIMEOUT_MS}`,
+      // An empty --grep-invert would match (and thus drop) every test, so
+      // the flag only appears when there are titles to skip.
+      ...(HANGING_TESTS.length
+        ? [`--grep-invert=${HANGING_TESTS.map((h) => h.pattern).join("|")}`]
+        : []),
       "--reporter=json",
     ],
     { cwd: pwRepo, env },
   );
+  clearInterval(heartbeat);
+  const elapsedSeconds = Math.round((Date.now() - startedAt) / 1_000);
+  // A globally-timed-out run still writes a valid JSON report with the
+  // remaining tests marked "did not run", which would otherwise diff as a
+  // false green — report the hang instead of trusting the partial report.
+  if (/Timed out waiting .* for the test suite/.test(output)) {
+    return { hung: true, elapsedSeconds };
+  }
   let report;
   try {
     report = JSON.parse(fs.readFileSync(jsonOut, "utf8"));
   } catch {
     console.error(output.slice(-4000));
     throw new Error(
-      `${mode}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
+      `${mode} ${spec}: no JSON report produced (exit code ${code}) — harness broken, see output above`,
     );
   }
-  return collectFailures(report);
+  const stats = report.stats ?? {};
+  const totalRun =
+    (stats.expected ?? 0) +
+    (stats.unexpected ?? 0) +
+    (stats.flaky ?? 0) +
+    (stats.skipped ?? 0);
+  return { failures: collectFailures(report), totalRun, elapsedSeconds };
 }
 
 const args = parseArgs(process.argv.slice(2));
@@ -188,41 +294,99 @@ if (!fs.existsSync(ibrDist)) {
 
 await applyFixturePatch(pwRepo);
 
+if (!args.specs) {
+  const { included, excluded } = discoverSpecs(pwRepo);
+  args.specs = included;
+  console.log(`Excluded specs (${excluded.length}):`);
+  for (const [name, reason] of excluded) {
+    console.log(`  [excluded] ${name} — ${reason}`);
+  }
+}
+if (args.shard) {
+  // Round-robin slice so slow specs spread across shards.
+  args.specs = args.specs.filter(
+    (_, i) => i % args.shard.total === args.shard.index - 1,
+  );
+  console.log(
+    `Shard ${args.shard.index}/${args.shard.total}: ${args.specs.length} specs`,
+  );
+}
+
 console.log(`Specs: ${args.specs.join(", ")}`);
-console.log("Running baseline (built-in blob reporter)...");
-const blobFailures = await runMode({
-  pwRepo,
-  specs: args.specs,
-  workers: args.workers,
-  mode: "blob",
-});
-console.log(
-  `Baseline failures (env noise, cancelled in diff): ${blobFailures.size}`,
-);
-for (const key of [...blobFailures].sort()) {
-  console.log(`  [baseline] ${key}`);
+for (const { pattern, reason } of HANGING_TESTS) {
+  console.log(`  [skipped-by-title] /${pattern}/ — ${reason}`);
 }
+// Each spec runs both modes back to back as its own bounded invocation
+// (see SPEC_TIMEOUT_MS); a hang or count mismatch quarantines that spec and
+// the sweep continues.
+const ibrOnly = [];
+const hungSpecs = [];
+const mismatchedSpecs = [];
+let baselineNoise = 0;
 
-console.log("Running through IBR adapter...");
-const ibrFailures = await runMode({
-  pwRepo,
-  specs: args.specs,
-  workers: args.workers,
-  mode: "ibr",
-  ibrDist,
-});
+for (const spec of args.specs) {
+  const blob = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "blob",
+  });
+  if (blob.hung) {
+    console.log(`  [hung] ${spec} (blob mode, ${blob.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (blob mode)`);
+    continue;
+  }
+  const ibr = await runMode({
+    pwRepo,
+    spec,
+    workers: args.workers,
+    mode: "ibr",
+    ibrDist,
+  });
+  if (ibr.hung) {
+    console.log(`  [hung] ${spec} (ibr mode, ${ibr.elapsedSeconds}s)`);
+    hungSpecs.push(`${spec} (ibr mode)`);
+    continue;
+  }
+  if (blob.totalRun === 0 || blob.totalRun !== ibr.totalRun) {
+    console.log(
+      `  [mismatch] ${spec}: blob ran ${blob.totalRun}, ibr ran ${ibr.totalRun}`,
+    );
+    mismatchedSpecs.push(spec);
+    continue;
+  }
+  const specIbrOnly = [...ibr.failures]
+    .filter((key) => !blob.failures.has(key))
+    .sort();
+  baselineNoise += blob.failures.size;
+  ibrOnly.push(...specIbrOnly);
+  console.log(
+    `  ${spec}: ${blob.totalRun} tests, ` +
+      `${blob.failures.size} baseline noise, ${specIbrOnly.length} ibr-only ` +
+      `(blob ${blob.elapsedSeconds}s, ibr ${ibr.elapsedSeconds}s)`,
+  );
+}
 
-const ibrOnly = [...ibrFailures].filter((key) => !blobFailures.has(key)).sort();
+console.log(
+  `\nBaseline failures (env noise, cancelled in diff): ${baselineNoise}`,
+);
 console.log(`IBR-only failures: ${ibrOnly.length}`);
-for (const key of ibrOnly) {
+for (const key of ibrOnly.sort()) {
   console.log(`  [ibr-only] ${key}`);
 }
+for (const spec of hungSpecs) {
+  console.log(`  [hung] ${spec}`);
+}
+for (const spec of mismatchedSpecs) {
+  console.log(`  [mismatch] ${spec}`);
+}
 
-if (ibrOnly.length > 0) {
+if (ibrOnly.length > 0 || hungSpecs.length > 0 || mismatchedSpecs.length > 0) {
   console.error(
-    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR. ` +
-      `The job stays red until IBR reaches blob parity (known gaps: onStdIO, ` +
-      `onError, result-level annotations, attachment bodies).`,
+    `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR, ` +
+      `${hungSpecs.length} spec run(s) hung, ${mismatchedSpecs.length} ` +
+      `spec(s) mismatched. The job stays red until IBR reaches blob parity ` +
+      `on these specs.`,
   );
   process.exit(1);
 }