@empiricalrun/playwright-utils 0.58.4 → 0.58.5

package/dist/reporter/parity-utils.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.normalizeJsonReport = normalizeJsonReport;
 exports.stepSignatures = stepSignatures;
+exports.stepAttachmentsAtStepEnd = stepAttachmentsAtStepEnd;
+exports.configAndProjects = configAndProjects;
 exports.missingEventMethods = missingEventMethods;
 const ANSI_RE = /\u001b\[[0-9;]*m/g;
 function normalizeText(text, cwd) {
@@ -93,11 +95,81 @@ function stepSignatures(events) {
             continue;
         }
         const list = signatures.get(key) ?? [];
-        list.push(`${params.step.category}:${params.step.title}`);
+        // The location is part of the signature: it travels relative to rootDir
+        // and the merged HTML report renders it as-is.
+        const location = params.step.location
+            ? `${params.step.location.file}:${params.step.location.line}`
+            : "";
+        list.push(`${params.step.category}:${params.step.title}@${location}`);
         signatures.set(key, list);
     }
     return signatures;
 }
+/**
+ * What a reporter would see for `step.attachments` DURING onStepEnd, replayed
+ * from the raw event stream the way teleReceiver does it: onAttach pushes onto
+ * `result.attachments`, and a step end carries indices into that array
+ * (TeleTestStep.attachments resolves them at access time). An attachment that
+ * only arrives at test end therefore reads as `<missing>` here.
+ *
+ * Keyed by `testId:retry`; each entry is one `title -> [names]` line per
+ * onStepEnd, in stream order.
+ */
+function stepAttachmentsAtStepEnd(events) {
+    const resultKey = new Map(); // resultId -> testId:retry
+    const attachmentNames = new Map(); // resultId -> names so far
+    const stepTitles = new Map(); // stepId -> title
+    const views = new Map();
+    for (const event of events) {
+        if (event.method === "onTestBegin") {
+            const params = event.params;
+            resultKey.set(params.result.id, `${params.testId}:${params.result.retry}`);
+            attachmentNames.set(params.result.id, []);
+        }
+        else if (event.method === "onAttach") {
+            const params = event.params;
+            const names = attachmentNames.get(params.resultId);
+            names?.push(...params.attachments.map((a) => a.name));
+        }
+        else if (event.method === "onStepBegin") {
+            const params = event.params;
+            stepTitles.set(params.step.id, params.step.title);
+        }
+        else if (event.method === "onStepEnd") {
+            const params = event.params;
+            const key = resultKey.get(params.resultId);
+            if (!key) {
+                continue;
+            }
+            const names = attachmentNames.get(params.resultId) ?? [];
+            const resolved = (params.step.attachments ?? []).map((index) => names[index] ?? "<missing>");
+            const list = views.get(key) ?? [];
+            list.push(`${stepTitles.get(params.step.id) ?? "?"} -> [${resolved.join(", ")}]`);
+            views.set(key, list);
+        }
+    }
+    return views;
+}
+/**
+ * The onConfigure config payload plus every onProject project payload, keyed
+ * by project name. The merged report's FullConfig/FullProject is rebuilt from
+ * exactly these, and normalizeJsonReport compares only the suite/stats body —
+ * so this is where config and project serialization drift shows up.
+ */
+function configAndProjects(events) {
+    let config;
+    const projects = {};
+    for (const event of events) {
+        if (event.method === "onConfigure") {
+            config = event.params.config;
+        }
+        else if (event.method === "onProject") {
+            const project = event.params.project;
+            projects[project.name] = project;
+        }
+    }
+    return { config, projects };
+}
 /** Event method names present in `reference` but absent from `candidate`. */
 function missingEventMethods(reference, candidate) {
     const candidateMethods = new Set(candidate.map((e) => e.method));

package/dist/reporter/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAKnC,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAqB;IAEzC,YAAY,MAAM,EAAE,cAAc,EASjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAEpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAKnC,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAqB;IAEzC,YAAY,MAAM,EAAE,cAAc,EASjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CA4BpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C"}

package/dist/reporter/uploader.js

@@ -100,7 +100,28 @@ class Uploader {
         await (0, r2_uploader_1.sendTaskToQueue)(uploadTask);
     }
     async waitForUploads() {
-        await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+        const { waiting, executing } = (0, r2_uploader_1.getTaskQueueStats)();
+        const pending = waiting + executing;
+        if (pending === 0) {
+            await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+            return;
+        }
+        // This drain runs after the last test with no other output (observed at
+        // 4+ minutes on spot runs), so make it visible: count up front, progress
+        // every 30s, duration at the end.
+        const start = Date.now();
+        console.log(`[Uploader] Waiting for ${pending} pending uploads (${executing} executing, ${waiting} queued)...`);
+        const progress = setInterval(() => {
+            const stats = (0, r2_uploader_1.getTaskQueueStats)();
+            console.log(`[Uploader] Still draining: ${stats.executing + stats.waiting} uploads pending (${stats.executing} executing, ${stats.waiting} queued)`);
+        }, 30_000);
+        try {
+            await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+        }
+        finally {
+            clearInterval(progress);
+        }
+        console.log(`[Uploader] Pending uploads drained in ${((Date.now() - start) / 1000).toFixed(1)}s`);
     }
 }
 exports.Uploader = Uploader;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.4",
+  "version": "0.58.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -56,7 +56,7 @@
     "ts-morph": "^23.0.0",
     "@empiricalrun/cua": "^0.4.1",
     "@empiricalrun/llm": "^0.27.0",
-    "@empiricalrun/r2-uploader": "^0.10.1",
+    "@empiricalrun/r2-uploader": "^0.10.2",
     "@empiricalrun/reporter": "^0.29.1"
   },
   "scripts": {

package/upstream-oracle/README.md

@@ -20,16 +20,45 @@ Playwright version-pin bump).
 
 - `run.mjs` — orchestrator: applies the patch, runs the curated specs in both
   modes, fails on any IBR-only failure.
-- `ibr-adapter.js` — bridges IBR's env/paths contract to the blob reporter's
-  (config.shard → SHARD_INDEX env, zip relocation to `blob-report/`).
+- `ibr-adapter.js` — bridges IBR's output contract to the blob reporter's: it
+  mirrors blob's zip name (`_defaultReportName`), output-dir resolution
+  (package root, not cwd) and pre-write cleanup, then relocates IBR's zip into
+  `blob-report/`. Sharding needs no bridge — IBR falls back to `config.shard`.
 - `intermediate-mode-ibr.patch` — one-line upstream fixture change making the
   intermediate reporter selectable via `PWTEST_IBR_ADAPTER`.
 
-The job is EXPECTED to stay red until IBR reaches parity: today ~55 upstream
-tests fail only through IBR, all tracing to the known gaps (`onStdIO`,
-`onError`, result-level annotations, attachment bodies — encoded as `it.fails`
-in `src/reporter/blob-ibr-parity.test.ts`). The failure count shrinking to
-zero is the signal that the built-in blob reporter is safe to drop.
+The failure count reaching zero is the signal that the built-in blob reporter
+is safe to drop. It got there over #7093/#7094/#7095 (the parity gaps and the
+full-suite sweep) and then #7110/#7111/#7114/this PR, which took the full
+sweep from 19 IBR-only failures to 0.
+
+### Expected failures
+
+`EXPECTED_IBR_ONLY` in `run.mjs` holds upstream tests that fail for reasons
+outside the blob format — artifacts of injecting an external reporter into
+upstream's harness. Each entry carries the reason it can never pass and is
+printed at startup; the count is reported separately and does not turn the job
+red. An entry whose spec runs and PASSES is reported as `[stale-expected]` and
+fails the run, so the list cannot quietly absorb tests that have been fixed.
+Anything that could be fixed does not belong there.
+
+### Why two slow specs are excluded
+
+`only-changed.spec.ts` and `web-server.spec.ts` are in `EXCLUSION_RULES` for a
+different reason from the rest: cost, not scope. The oracle runs ONE spec per
+invocation, and Playwright keeps a file's tests on a single worker
+(`fullyParallel` defaults to false), so a long file runs strictly serially no
+matter what `--workers` says. Measured, `only-changed` gets through 7 of 13
+tests in 7 minutes — steady progress, no wedge — and `web-server` has 39
+server-booting tests. Neither fits the 10-minute `SPEC_TIMEOUT_MS`, so both
+used to burn the cap, get quarantined, and contribute nothing but a hang that
+kept the job red.
+
+If the sweep's wall-clock ever needs fixing properly, the lever is `run.mjs`
+running specs sequentially: several invocations could run concurrently with no
+loss of accuracy (each spec's blob/IBR pair still compares like for like),
+once each gets its own `--output` dir instead of sharing
+`<pwRepo>/test-results`.
 
 ## Running locally
 

package/upstream-oracle/ibr-adapter.js

@@ -45,6 +45,55 @@ class IBRBlobAdapter extends IBR {
     super();
     this._adapterOptions = options;
     this._adapterCwd = process.cwd();
+    this._adapterShard = null;
+  }
+
+  onBegin(config, suite) {
+    this._adapterShard = config.shard;
+    super.onBegin(config, suite);
+  }
+
+  // Mirrors resolveOutputFile('BLOB', …) + resolveReporterOutputPath: an
+  // explicit outputDir resolves against the config dir, and the DEFAULT
+  // blob-report hangs off the nearest package.json at or above the config
+  // dir (process.cwd() only as a last resort). Upstream drives the CLI from
+  // a nested dir in several tests and then merges <packageRoot>/blob-report,
+  // so resolving against the cwd puts the zip where nothing looks for it.
+  _resolveOutputDir() {
+    const configDir = this._adapterOptions.configDir || this._adapterCwd;
+    if (this._adapterOptions.outputDir) {
+      return path.resolve(configDir, this._adapterOptions.outputDir);
+    }
+    return path.resolve(this._packageRoot(configDir), "blob-report");
+  }
+
+  _packageRoot(configDir) {
+    let dir = path.resolve(configDir);
+    for (;;) {
+      if (fs.existsSync(path.join(dir, "package.json"))) {
+        return dir;
+      }
+      const parent = path.dirname(dir);
+      if (parent === dir) {
+        return this._adapterCwd;
+      }
+      dir = parent;
+    }
+  }
+
+  // Mirrors blob's _defaultReportName: the shard suffix only appears on a
+  // sharded run, zero-padded to the width of the shard total. Upstream tests
+  // assert on the file name (reporter-blob.spec's "preserve steps in html
+  // report" expects exactly ['report.zip']).
+  _defaultReportName() {
+    if (!this._adapterShard) {
+      return "report.zip";
+    }
+    const padded = `${this._adapterShard.current}`.padStart(
+      `${this._adapterShard.total}`.length,
+      "0",
+    );
+    return `report-${padded}.zip`;
   }
 
   async onEnd(result) {
@@ -59,11 +108,17 @@ class IBRBlobAdapter extends IBR {
       // Mirrors blob: no zip when there was nothing to report.
       return;
     }
-    const outputDir = path.resolve(
-      this._adapterCwd,
-      this._adapterOptions.outputDir || "blob-report",
-    );
-    const fileName = this._adapterOptions.fileName || `report-${shard}.zip`;
+    const outputDir = this._resolveOutputDir();
+    const fileName = this._adapterOptions.fileName || this._defaultReportName();
+    // Mirrors blob's _prepareOutputFile, down to the opt-out: it wipes the
+    // output dir before writing unless PWTEST_BLOB_DO_NOT_REMOVE is set.
+    // Upstream tests merge whatever sits in blob-report/, and they run
+    // several shards from one baseDir — those that want the shards to
+    // accumulate set the env var, the rest (shard.spec's "shard 2/3 then
+    // 3/3") expect each run to start from an empty dir.
+    if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE) {
+      fs.rmSync(outputDir, { recursive: true, force: true });
+    }
     fs.mkdirSync(outputDir, { recursive: true });
     fs.copyFileSync(src, path.join(outputDir, fileName));
   }

package/upstream-oracle/run.mjs

@@ -12,7 +12,10 @@
  * Both modes run the same spec files; environment noise cancels in the
  * baseline diff. Any IBR-only failure fails the run — the job stays red
  * until IBR reaches parity on these specs. It staying green is the signal
- * that the built-in blob reporter is safe to drop.
+ * that the built-in blob reporter is safe to drop. The one carve-out is
+ * EXPECTED_IBR_ONLY below: failures caused by injecting an external reporter
+ * rather than by the blob format, each with the reason it cannot pass, and
+ * each policed by a staleness check so the list cannot rot.
  *
  * Usage:
  *   node run.mjs --pw-repo <upstream clone at the pinned tag>
@@ -65,8 +68,51 @@ const EXCLUSION_RULES = [
       "semantics are covered by reporter-blob.spec.ts) and they busy-loop " +
       "workers under load, wedging the sweep",
   ],
+  // The two specs below are SLOW, not wedged — measured, they make steady
+  // progress and simply do not fit in SPEC_TIMEOUT_MS. The oracle runs one
+  // spec per invocation and Playwright keeps a file's tests on a single
+  // worker (fullyParallel defaults to false), so a long file runs strictly
+  // serially however many --workers it is given. Both were already
+  // contributing no signal, just a 10-minute timeout each.
+  [
+    (name) => name === "only-changed.spec.ts",
+    "git-based run selection (--only-changed), not report format; 13 tests " +
+      "that each shell out to git take ~13min serially here",
+  ],
+  [
+    (name) => name === "web-server.spec.ts",
+    "webServer config and lifecycle, not report format; 39 tests that each " +
+      "boot and tear down a server take longer still",
+  ],
 ];
 
+// Upstream tests that fail through IBR for reasons OUTSIDE the blob format —
+// artifacts of injecting an external reporter into upstream's harness, not
+// gaps in what IBR writes. They are reported and subtracted from the failure
+// count instead of turning the job red.
+//
+// Kept honest by the staleness check at the end: an entry whose spec ran and
+// did NOT fail is reported and fails the run, so a fixed test cannot sit here
+// pretending to be unfixable. Add an entry only with the reason it can never
+// pass; anything that could be fixed belongs in the failure list.
+const EXPECTED_IBR_ONLY = [
+  {
+    key:
+      "cache.spec.ts:53 › should automatically clean cached versions of a " +
+      "changed file",
+    reason:
+      "asserts the transform cache holds exactly one directory; loading IBR " +
+      "from a path runs its bundle through the same transform pipeline, so " +
+      "the cache gains a directory per module (32 observed). The built-in " +
+      "blob reporter ships inside Playwright and never enters that cache.",
+  },
+];
+
+/** Spec file name out of a failure key ("cache.spec.ts:53 › title"). */
+function specOfKey(key) {
+  return key.slice(0, key.indexOf(":"));
+}
+
 // Tests skipped by title via --grep-invert. Reserved for tests that WEDGE
 // the sweep — a busy-looped worker no per-test timeout can interrupt — so
 // exclusion is surgical (exact title match) and every entry carries the
@@ -316,12 +362,18 @@ console.log(`Specs: ${args.specs.join(", ")}`);
 for (const { pattern, reason } of HANGING_TESTS) {
   console.log(`  [skipped-by-title] /${pattern}/ — ${reason}`);
 }
+for (const { key, reason } of EXPECTED_IBR_ONLY) {
+  console.log(`  [expected-ibr-only] ${key} — ${reason}`);
+}
 // Each spec runs both modes back to back as its own bounded invocation
 // (see SPEC_TIMEOUT_MS); a hang or count mismatch quarantines that spec and
 // the sweep continues.
 const ibrOnly = [];
 const hungSpecs = [];
 const mismatchedSpecs = [];
+const expectedByKey = new Map(EXPECTED_IBR_ONLY.map((e) => [e.key, e]));
+const matchedExpected = new Set();
+const comparedSpecs = new Set();
 let baselineNoise = 0;
 
 for (const spec of args.specs) {
@@ -355,21 +407,31 @@ for (const spec of args.specs) {
     mismatchedSpecs.push(spec);
     continue;
   }
+  comparedSpecs.add(spec);
   const specIbrOnly = [...ibr.failures]
     .filter((key) => !blob.failures.has(key))
     .sort();
   baselineNoise += blob.failures.size;
-  ibrOnly.push(...specIbrOnly);
+  const specExpected = specIbrOnly.filter((key) => expectedByKey.has(key));
+  for (const key of specExpected) {
+    matchedExpected.add(key);
+  }
+  const specUnexpected = specIbrOnly.filter((key) => !expectedByKey.has(key));
+  ibrOnly.push(...specUnexpected);
   console.log(
     `  ${spec}: ${blob.totalRun} tests, ` +
-      `${blob.failures.size} baseline noise, ${specIbrOnly.length} ibr-only ` +
-      `(blob ${blob.elapsedSeconds}s, ibr ${ibr.elapsedSeconds}s)`,
+      `${blob.failures.size} baseline noise, ${specUnexpected.length} ibr-only` +
+      (specExpected.length ? `, ${specExpected.length} expected` : "") +
+      ` (blob ${blob.elapsedSeconds}s, ibr ${ibr.elapsedSeconds}s)`,
   );
 }
 
 console.log(
   `\nBaseline failures (env noise, cancelled in diff): ${baselineNoise}`,
 );
+console.log(
+  `Expected IBR-only failures (not counted): ${matchedExpected.size}`,
+);
 console.log(`IBR-only failures: ${ibrOnly.length}`);
 for (const key of ibrOnly.sort()) {
   console.log(`  [ibr-only] ${key}`);
@@ -381,12 +443,27 @@ for (const spec of mismatchedSpecs) {
   console.log(`  [mismatch] ${spec}`);
 }
 
-if (ibrOnly.length > 0 || hungSpecs.length > 0 || mismatchedSpecs.length > 0) {
+// An expected failure whose spec ran and passed means the list is stale —
+// that is a fix to celebrate by deleting the entry, not a silent pass.
+const staleExpected = EXPECTED_IBR_ONLY.filter(
+  ({ key }) => comparedSpecs.has(specOfKey(key)) && !matchedExpected.has(key),
+);
+for (const { key } of staleExpected) {
+  console.log(`  [stale-expected] ${key} — now passes; drop it from the list`);
+}
+
+if (
+  ibrOnly.length > 0 ||
+  hungSpecs.length > 0 ||
+  mismatchedSpecs.length > 0 ||
+  staleExpected.length > 0
+) {
   console.error(
     `\nFAIL: ${ibrOnly.length} upstream test(s) fail only through IBR, ` +
       `${hungSpecs.length} spec run(s) hung, ${mismatchedSpecs.length} ` +
-      `spec(s) mismatched. The job stays red until IBR reaches blob parity ` +
-      `on these specs.`,
+      `spec(s) mismatched, ${staleExpected.length} stale expected-failure ` +
+      `entr${staleExpected.length === 1 ? "y" : "ies"}. The job stays red ` +
+      `until IBR reaches blob parity on these specs.`,
   );
   process.exit(1);
 }