@empiricalrun/playwright-utils 0.58.3 → 0.58.5

package/dist/reporter/parity-utils.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.normalizeJsonReport = normalizeJsonReport;
 exports.stepSignatures = stepSignatures;
+exports.stepAttachmentsAtStepEnd = stepAttachmentsAtStepEnd;
+exports.configAndProjects = configAndProjects;
 exports.missingEventMethods = missingEventMethods;
 const ANSI_RE = /\u001b\[[0-9;]*m/g;
 function normalizeText(text, cwd) {
@@ -93,11 +95,81 @@ function stepSignatures(events) {
             continue;
         }
         const list = signatures.get(key) ?? [];
-        list.push(`${params.step.category}:${params.step.title}`);
+        // The location is part of the signature: it travels relative to rootDir
+        // and the merged HTML report renders it as-is.
+        const location = params.step.location
+            ? `${params.step.location.file}:${params.step.location.line}`
+            : "";
+        list.push(`${params.step.category}:${params.step.title}@${location}`);
         signatures.set(key, list);
     }
     return signatures;
 }
+/**
+ * What a reporter would see for `step.attachments` DURING onStepEnd, replayed
+ * from the raw event stream the way teleReceiver does it: onAttach pushes onto
+ * `result.attachments`, and a step end carries indices into that array
+ * (TeleTestStep.attachments resolves them at access time). An attachment that
+ * only arrives at test end therefore reads as `<missing>` here.
+ *
+ * Keyed by `testId:retry`; each entry is one `title -> [names]` line per
+ * onStepEnd, in stream order.
+ */
+function stepAttachmentsAtStepEnd(events) {
+    const resultKey = new Map(); // resultId -> testId:retry
+    const attachmentNames = new Map(); // resultId -> names so far
+    const stepTitles = new Map(); // stepId -> title
+    const views = new Map();
+    for (const event of events) {
+        if (event.method === "onTestBegin") {
+            const params = event.params;
+            resultKey.set(params.result.id, `${params.testId}:${params.result.retry}`);
+            attachmentNames.set(params.result.id, []);
+        }
+        else if (event.method === "onAttach") {
+            const params = event.params;
+            const names = attachmentNames.get(params.resultId);
+            names?.push(...params.attachments.map((a) => a.name));
+        }
+        else if (event.method === "onStepBegin") {
+            const params = event.params;
+            stepTitles.set(params.step.id, params.step.title);
+        }
+        else if (event.method === "onStepEnd") {
+            const params = event.params;
+            const key = resultKey.get(params.resultId);
+            if (!key) {
+                continue;
+            }
+            const names = attachmentNames.get(params.resultId) ?? [];
+            const resolved = (params.step.attachments ?? []).map((index) => names[index] ?? "<missing>");
+            const list = views.get(key) ?? [];
+            list.push(`${stepTitles.get(params.step.id) ?? "?"} -> [${resolved.join(", ")}]`);
+            views.set(key, list);
+        }
+    }
+    return views;
+}
+/**
+ * The onConfigure config payload plus every onProject project payload, keyed
+ * by project name. The merged report's FullConfig/FullProject is rebuilt from
+ * exactly these, and normalizeJsonReport compares only the suite/stats body —
+ * so this is where config and project serialization drift shows up.
+ */
+function configAndProjects(events) {
+    let config;
+    const projects = {};
+    for (const event of events) {
+        if (event.method === "onConfigure") {
+            config = event.params.config;
+        }
+        else if (event.method === "onProject") {
+            const project = event.params.project;
+            projects[project.name] = project;
+        }
+    }
+    return { config, projects };
+}
 /** Event method names present in `reference` but absent from `candidate`. */
 function missingEventMethods(reference, candidate) {
     const candidateMethods = new Set(candidate.map((e) => e.method));

package/dist/reporter/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAKnC,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAqB;IAEzC,YAAY,MAAM,EAAE,cAAc,EASjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAEpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAKnC,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAqB;IAEzC,YAAY,MAAM,EAAE,cAAc,EASjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CA4BpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C"}

package/dist/reporter/uploader.js

@@ -100,7 +100,28 @@ class Uploader {
         await (0, r2_uploader_1.sendTaskToQueue)(uploadTask);
     }
     async waitForUploads() {
-        await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+        const { waiting, executing } = (0, r2_uploader_1.getTaskQueueStats)();
+        const pending = waiting + executing;
+        if (pending === 0) {
+            await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+            return;
+        }
+        // This drain runs after the last test with no other output (observed at
+        // 4+ minutes on spot runs), so make it visible: count up front, progress
+        // every 30s, duration at the end.
+        const start = Date.now();
+        console.log(`[Uploader] Waiting for ${pending} pending uploads (${executing} executing, ${waiting} queued)...`);
+        const progress = setInterval(() => {
+            const stats = (0, r2_uploader_1.getTaskQueueStats)();
+            console.log(`[Uploader] Still draining: ${stats.executing + stats.waiting} uploads pending (${stats.executing} executing, ${stats.waiting} queued)`);
+        }, 30_000);
+        try {
+            await (0, r2_uploader_1.waitForTaskQueueToFinish)();
+        }
+        finally {
+            clearInterval(progress);
+        }
+        console.log(`[Uploader] Pending uploads drained in ${((Date.now() - start) / 1000).toFixed(1)}s`);
     }
 }
 exports.Uploader = Uploader;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.3",
+  "version": "0.58.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -41,7 +41,7 @@
     "openapi-typescript": "7.6.1",
     "playwright-core": "1.62.1",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.15.3"
+    "@empiricalrun/shared-types": "0.15.4"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
@@ -54,9 +54,9 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/llm": "^0.27.0",
     "@empiricalrun/cua": "^0.4.1",
-    "@empiricalrun/r2-uploader": "^0.10.1",
+    "@empiricalrun/llm": "^0.27.0",
+    "@empiricalrun/r2-uploader": "^0.10.2",
     "@empiricalrun/reporter": "^0.29.1"
   },
   "scripts": {

package/upstream-oracle/README.md

@@ -20,16 +20,45 @@ Playwright version-pin bump).
 
 - `run.mjs` — orchestrator: applies the patch, runs the curated specs in both
   modes, fails on any IBR-only failure.
-- `ibr-adapter.js` — bridges IBR's env/paths contract to the blob reporter's
-  (config.shard → SHARD_INDEX env, zip relocation to `blob-report/`).
+- `ibr-adapter.js` — bridges IBR's output contract to the blob reporter's: it
+  mirrors blob's zip name (`_defaultReportName`), output-dir resolution
+  (package root, not cwd) and pre-write cleanup, then relocates IBR's zip into
+  `blob-report/`. Sharding needs no bridge — IBR falls back to `config.shard`.
 - `intermediate-mode-ibr.patch` — one-line upstream fixture change making the
   intermediate reporter selectable via `PWTEST_IBR_ADAPTER`.
 
-The job is EXPECTED to stay red until IBR reaches parity: today ~55 upstream
-tests fail only through IBR, all tracing to the known gaps (`onStdIO`,
-`onError`, result-level annotations, attachment bodies — encoded as `it.fails`
-in `src/reporter/blob-ibr-parity.test.ts`). The failure count shrinking to
-zero is the signal that the built-in blob reporter is safe to drop.
+The failure count reaching zero is the signal that the built-in blob reporter
+is safe to drop. It got there over #7093/#7094/#7095 (the parity gaps and the
+full-suite sweep) and then #7110/#7111/#7114/this PR, which took the full
+sweep from 19 IBR-only failures to 0.
+
+### Expected failures
+
+`EXPECTED_IBR_ONLY` in `run.mjs` holds upstream tests that fail for reasons
+outside the blob format — artifacts of injecting an external reporter into
+upstream's harness. Each entry carries the reason it can never pass and is
+printed at startup; the count is reported separately and does not turn the job
+red. An entry whose spec runs and PASSES is reported as `[stale-expected]` and
+fails the run, so the list cannot quietly absorb tests that have been fixed.
+Anything that could be fixed does not belong there.
+
+### Why two slow specs are excluded
+
+`only-changed.spec.ts` and `web-server.spec.ts` are in `EXCLUSION_RULES` for a
+different reason from the rest: cost, not scope. The oracle runs ONE spec per
+invocation, and Playwright keeps a file's tests on a single worker
+(`fullyParallel` defaults to false), so a long file runs strictly serially no
+matter what `--workers` says. Measured, `only-changed` gets through 7 of 13
+tests in 7 minutes — steady progress, no wedge — and `web-server` has 39
+server-booting tests. Neither fits the 10-minute `SPEC_TIMEOUT_MS`, so both
+used to burn the cap, get quarantined, and contribute nothing but a hang that
+kept the job red.
+
+If the sweep's wall-clock ever needs fixing properly, the lever is `run.mjs`
+running specs sequentially: several invocations could run concurrently with no
+loss of accuracy (each spec's blob/IBR pair still compares like for like),
+once each gets its own `--output` dir instead of sharing
+`<pwRepo>/test-results`.
 
 ## Running locally
 

package/upstream-oracle/ibr-adapter.js

@@ -45,6 +45,55 @@ class IBRBlobAdapter extends IBR {
     super();
     this._adapterOptions = options;
     this._adapterCwd = process.cwd();
+    this._adapterShard = null;
+  }
+
+  onBegin(config, suite) {
+    this._adapterShard = config.shard;
+    super.onBegin(config, suite);
+  }
+
+  // Mirrors resolveOutputFile('BLOB', …) + resolveReporterOutputPath: an
+  // explicit outputDir resolves against the config dir, and the DEFAULT
+  // blob-report hangs off the nearest package.json at or above the config
+  // dir (process.cwd() only as a last resort). Upstream drives the CLI from
+  // a nested dir in several tests and then merges <packageRoot>/blob-report,
+  // so resolving against the cwd puts the zip where nothing looks for it.
+  _resolveOutputDir() {
+    const configDir = this._adapterOptions.configDir || this._adapterCwd;
+    if (this._adapterOptions.outputDir) {
+      return path.resolve(configDir, this._adapterOptions.outputDir);
+    }
+    return path.resolve(this._packageRoot(configDir), "blob-report");
+  }
+
+  _packageRoot(configDir) {
+    let dir = path.resolve(configDir);
+    for (;;) {
+      if (fs.existsSync(path.join(dir, "package.json"))) {
+        return dir;
+      }
+      const parent = path.dirname(dir);
+      if (parent === dir) {
+        return this._adapterCwd;
+      }
+      dir = parent;
+    }
+  }
+
+  // Mirrors blob's _defaultReportName: the shard suffix only appears on a
+  // sharded run, zero-padded to the width of the shard total. Upstream tests
+  // assert on the file name (reporter-blob.spec's "preserve steps in html
+  // report" expects exactly ['report.zip']).
+  _defaultReportName() {
+    if (!this._adapterShard) {
+      return "report.zip";
+    }
+    const padded = `${this._adapterShard.current}`.padStart(
+      `${this._adapterShard.total}`.length,
+      "0",
+    );
+    return `report-${padded}.zip`;
   }
 
   async onEnd(result) {
@@ -59,11 +108,17 @@ class IBRBlobAdapter extends IBR {
       // Mirrors blob: no zip when there was nothing to report.
       return;
     }
-    const outputDir = path.resolve(
-      this._adapterCwd,
-      this._adapterOptions.outputDir || "blob-report",
-    );
-    const fileName = this._adapterOptions.fileName || `report-${shard}.zip`;
+    const outputDir = this._resolveOutputDir();
+    const fileName = this._adapterOptions.fileName || this._defaultReportName();
+    // Mirrors blob's _prepareOutputFile, down to the opt-out: it wipes the
+    // output dir before writing unless PWTEST_BLOB_DO_NOT_REMOVE is set.
+    // Upstream tests merge whatever sits in blob-report/, and they run
+    // several shards from one baseDir — those that want the shards to
+    // accumulate set the env var, the rest (shard.spec's "shard 2/3 then
+    // 3/3") expect each run to start from an empty dir.
+    if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE) {
+      fs.rmSync(outputDir, { recursive: true, force: true });
+    }
     fs.mkdirSync(outputDir, { recursive: true });
     fs.copyFileSync(src, path.join(outputDir, fileName));
   }