@empiricalrun/playwright-utils 0.58.1 → 0.58.3

package/dist/reporter/incremental-blob-reporter.js

@@ -28,9 +28,10 @@ class IncrementalBlobReporter {
     _completedTestIds = new Set();
     _completedResultIds = new Set();
     _testRetries = new Map(); // testId → configured retries
-    _testResults = new Map(); // testId → retryIndex → {resultId, status, expectedStatus}
+    _testResults = new Map(); // testId → retryIndex → RecordedRetry
     _startTime = Date.now();
     _finalizePromise = null;
+    _rootDir = process.cwd();
     constructor() {
         if (process.env.SHARD_INDEX && process.env.TOTAL_SHARDS) {
             this._shardIndex = Number.parseInt(process.env.SHARD_INDEX, 10);
@@ -101,51 +102,73 @@ class IncrementalBlobReporter {
         if (this._uploader) {
             await this._uploader.waitForUploads();
         }
-        this._finalizeReport(result, pruneIncompleteTests);
-        await this._writeZip();
+        const lines = this._buildFinalReportLines(result, pruneIncompleteTests);
+        await this._writeZip(lines);
         if (!upload) {
             logFinalizeComplete();
             return;
         }
         if (!this._uploader) {
-            console.warn("[IncrementalBlobReporterFailure] No uploader available, skipping upload");
+            console.warn("[IncrementalBlobReporter] No uploader available, skipping upload");
             logFinalizeComplete();
             return;
         }
         await this._uploader.uploadFile(this._zipPath, `blobs/incremental-report-${this._shardIndex}.zip`);
         await this._uploader.waitForUploads();
+        if (source === "onEnd") {
+            fs_1.default.writeFileSync(this._markerPath, JSON.stringify({ status: result.status, events: lines.length }), "utf8");
+            await this._uploader.uploadFile(this._markerPath, `blobs/incremental-report-${this._shardIndex}.zip.complete`);
+            await this._uploader.waitForUploads();
+        }
         console.log(`[IncrementalBlobReporter] Finalize and upload complete from ${source}`);
         if ((0, ibr_utils_1.isLocalTesting)()) {
             await (0, local_test_1.mergeForLocalTest)(this._currentWorkingDir, this._outputDir);
         }
         logFinalizeComplete();
     }
-    _finalizeReport(result, pruneIncompleteTests) {
+    /**
+     * Computes the lines to write, without touching `_reportLines`: recording
+     * continues after the SIGINT salvage, and the onEnd pass re-finalizes from
+     * the full stream — mutating here would make the second pass work from the
+     * first pass's pruned leftovers.
+     */
+    _buildFinalReportLines(result, pruneIncompleteTests) {
+        let lines = [...this._reportLines];
         if (pruneIncompleteTests) {
             // Determine which tests are fully complete (passed or all retries finished)
             this._computeCompletedSets();
             // Remove unrun tests from onProject suites
-            this._patchProjectSuites();
+            lines = this._patchProjectSuites(lines);
             // Remove events (onTestBegin, onStepBegin, etc.) for incomplete tests
-            this._removeIncompleteTestArtifacts();
+            lines = this._removeIncompleteTestArtifacts(lines);
         }
-        // Push onEnd directly. SIGINT marks the reporter finalized before this,
-        // so using _appendEvent would drop the synthetic onEnd event.
-        this._reportLines.push(JSON.stringify((0, lifecycle_events_1.buildOnEndEvent)(result)));
+        // The onEnd event belongs only in the written report, never in the
+        // in-memory stream — a later finalize pass appends its own.
+        lines.push(JSON.stringify((0, lifecycle_events_1.buildOnEndEvent)(result)));
+        return lines;
     }
     _computeCompletedSets() {
         for (const [testId, results] of this._testResults) {
             const maxRetries = this._testRetries.get(testId);
-            if ((0, reporter_1.areRetriesComplete)(maxRetries, results)) {
+            // Shutdown artifacts are not finished retries: an "interrupted" result
+            // is a test cut off mid-flight, and a post-signal "skipped" is a serial
+            // sibling Playwright marks off while shutting down. The resume shard
+            // re-runs both, so counting them as complete would put their results in
+            // the final zip alongside the resume shard's, duplicating tests in the
+            // merge. A post-signal passed/failed is real, though: the test finished
+            // its body and only its aftermath overlapped the shutdown.
+            const finished = new Map([...results].filter(([, r]) => r.status !== "interrupted" &&
+                !(r.postSignal && r.status === "skipped")));
+            if ((0, reporter_1.areRetriesComplete)(maxRetries, finished)) {
                 this._completedTestIds.add(testId);
-                for (const { resultId } of results.values()) {
+                for (const { resultId } of finished.values()) {
                     this._completedResultIds.add(resultId);
                 }
             }
         }
     }
-    _removeIncompleteTestArtifacts() {
-        this._reportLines = this._reportLines.filter((line) => {
+    _removeIncompleteTestArtifacts(lines) {
+        return lines.filter((line) => {
             const parsed = JSON.parse(line);
             const resultId = parsed.params?.resultId ?? parsed.params?.result?.id;
             if (!resultId) {
@@ -154,24 +177,26 @@ class IncrementalBlobReporter {
             return this._completedResultIds.has(resultId);
         });
     }
-    _patchProjectSuites() {
-        for (let i = 0; i < this._reportLines.length; i++) {
-            const line = this._reportLines[i];
+    _patchProjectSuites(lines) {
+        return lines.map((line) => {
             const parsed = JSON.parse(line);
             if (parsed.method !== "onProject") {
-                continue;
+                return line;
             }
             const project = parsed.params.project;
             const filteredSuites = project.suites
                 .map((suite) => (0, ibr_utils_1.filterSuiteByCompletedTests)(suite, this._completedTestIds))
                 .filter(Boolean);
             project.suites = filteredSuites;
-            this._reportLines[i] = JSON.stringify(parsed);
-        }
+            return JSON.stringify(parsed);
+        });
     }
     get _zipPath() {
         return path_1.default.join(this._outputDir, `incremental-report-${this._shardIndex}.zip`);
     }
+    get _markerPath() {
+        return `${this._zipPath}.complete`;
+    }
     get _jsonlPath() {
         return path_1.default.join(this._stagingDir, "report.jsonl");
     }
@@ -183,9 +208,10 @@ class IncrementalBlobReporter {
         fs_1.default.mkdirSync(this._outputDir, { recursive: true });
     }
     _appendEvent(event) {
-        if ((0, reporter_state_1.isFinalized)()) {
-            return false;
-        }
+        // Deliberately not gated on isFinalized(): the SIGINT salvage seals a
+        // *snapshot*, not the stream. Playwright keeps running afterEach/afterAll
+        // teardown after the signal, and those events feed the second finalize
+        // pass in onEnd.
         this._reportLines.push(JSON.stringify(event));
         return true;
     }
@@ -198,10 +224,10 @@ class IncrementalBlobReporter {
         }
         return allAppended;
     }
-    async _writeZip() {
+    async _writeZip(lines) {
         this._ensureDirs();
         // Write report.jsonl to the output directory
-        const jsonlContent = this._reportLines.join("\n") + "\n";
+        const jsonlContent = lines.join("\n") + "\n";
         fs_1.default.writeFileSync(this._jsonlPath, jsonlContent, "utf8");
         // Write _empirical_urls.json with attachment URL mappings
         const urlsJson = (0, ibr_utils_1.buildUrlsJson)(this._attachmentUrlMap);
@@ -209,10 +235,18 @@ class IncrementalBlobReporter {
         // Create zip from the staging directory
         const zipBuffer = await (0, zip_1.createZipFromDirectory)(this._stagingDir);
         fs_1.default.writeFileSync(this._zipPath, zipBuffer);
-        console.log(`[IncrementalBlobReporter] Zip updated: ${this._zipPath} (${this._reportLines.length} events, ${this._attachmentUrlMap.size} attachments)`);
+        console.log(`[IncrementalBlobReporter] Zip updated: ${this._zipPath} (${lines.length} events, ${this._attachmentUrlMap.size} attachments)`);
     }
     onBegin(config, suite) {
         this._ensureDirs();
+        this._rootDir = config.rootDir;
+        // The env vars win (production sets them), but a sharded run without
+        // them should still stamp the right shard into the blob metadata and
+        // the zip name instead of defaulting every shard to 1/1.
+        if (!process.env.SHARD_INDEX && config.shard) {
+            this._shardIndex = config.shard.current;
+            this._totalShards = config.shard.total;
+        }
         this._appendEvents((0, lifecycle_events_1.buildOnBeginEvents)(config, suite, this._currentWorkingDir, this._shardIndex, this._totalShards));
         console.log(`[IncrementalBlobReporter] Started with ${suite.allTests().length} tests`);
     }
@@ -220,7 +254,7 @@ class IncrementalBlobReporter {
         this._appendEvent((0, lifecycle_events_1.buildOnTestBeginEvent)(test, result, this._resultIdMap));
     }
     onTestEnd(test, result) {
-        const events = (0, lifecycle_events_1.buildOnTestEndEvents)(test, result, this._resultIdMap, this._stagingDir);
+        const events = (0, lifecycle_events_1.buildOnTestEndEvents)(test, result, this._resultIdMap, this._stagingDir, this._rootDir);
         if (this._appendEvents(events)) {
             if (!this._testRetries.has(test.id)) {
                 this._testRetries.set(test.id, test.retries);
@@ -234,21 +268,43 @@ class IncrementalBlobReporter {
                     resultId,
                     status: result.status,
                     expectedStatus: test.expectedStatus,
+                    // The SIGINT salvage has started; this result arrived during
+                    // interruption shutdown rather than the normal run.
+                    postSignal: this._finalizePromise !== null,
                 });
             }
         }
     }
+    onStdOut(chunk, test, result) {
+        this._appendEvent((0, lifecycle_events_1.buildOnStdIOEvent)("stdout", chunk, test, result, this._resultIdMap));
+    }
+    onStdErr(chunk, test, result) {
+        this._appendEvent((0, lifecycle_events_1.buildOnStdIOEvent)("stderr", chunk, test, result, this._resultIdMap));
+    }
+    onError(error, workerInfo) {
+        this._appendEvent((0, lifecycle_events_1.buildOnErrorEvent)(error, workerInfo));
+    }
     onStepBegin(test, result, step) {
         this._appendEvent((0, lifecycle_events_1.buildOnStepBeginEvent)(test, result, step, this._resultIdMap, this._stepIdMap));
     }
     onStepEnd(test, result, step) {
-        this._appendEvent((0, lifecycle_events_1.buildOnStepEndEvent)(test, result, step, this._resultIdMap, this._stepIdMap));
+        this._appendEvent((0, lifecycle_events_1.buildOnStepEndEvent)(test, result, step, this._resultIdMap, this._stepIdMap, this._rootDir));
     }
     async onEnd(result) {
         this._removeSignalHandler();
-        // If SIGINT already started the partial-report finalization, let that
-        // finish instead of appending a second onEnd or racing another zip write.
+        // Second phase of an interrupted run: the SIGINT pass salvaged a snapshot
+        // within milliseconds (SIGKILL insurance); by the time Playwright calls
+        // onEnd the teardown chain has finished, so re-finalize from the full
+        // stream and overwrite the salvage blob with the post-teardown report.
         if (this._finalizePromise) {
+            await this._finalizePromise;
+            this._finalizePromise = this._finalizeAndMaybeUpload({
+                result,
+                source: "onEnd",
+                markGlobalFinalized: false,
+                pruneIncompleteTests: true,
+                upload: true,
+            });
             await this._finalizePromise;
             return;
         }
@@ -257,9 +313,7 @@ class IncrementalBlobReporter {
             source: "onEnd",
             markGlobalFinalized: false,
             pruneIncompleteTests: false,
-            // Normal completion already has Playwright's blob/html reporters.
-            // Only the SIGINT path uploads this incremental fallback blob.
-            upload: false,
+            upload: true,
         });
         await this._finalizePromise;
         console.log(`[IncrementalBlobReporter] Finished with status: ${result.status}`);

package/dist/reporter/lifecycle-events.d.ts

@@ -1,4 +1,4 @@
-import type { FullConfig, FullProject, FullResult, Suite, TestCase, TestResult, TestStep } from "@playwright/test/reporter";
+import type { FullConfig, FullProject, FullResult, Suite, TestCase, TestError, TestResult, TestStep, WorkerInfo } from "@playwright/test/reporter";
 export type JsonEvent = {
     method: string;
     params?: object;
@@ -6,8 +6,10 @@ export type JsonEvent = {
 export declare function buildOnBeginEvents(config: FullConfig, suite: Suite, currentWorkingDir: string, shardIndex: number, totalShards: number): JsonEvent[];
 export declare function buildOnProjectEvent(config: FullConfig, project: FullProject, projectSuite: Suite, currentWorkingDir: string): JsonEvent;
 export declare function buildOnTestBeginEvent(test: TestCase, result: TestResult, resultIdMap: Map<string, string>): JsonEvent;
-export declare function buildOnTestEndEvents(test: TestCase, result: TestResult, resultIdMap: Map<string, string>, stagingDir: string): JsonEvent[];
+export declare function buildOnTestEndEvents(test: TestCase, result: TestResult, resultIdMap: Map<string, string>, stagingDir: string, rootDir: string): JsonEvent[];
+export declare function buildOnStdIOEvent(type: "stdout" | "stderr", chunk: string | Buffer, test: TestCase | undefined, result: TestResult | undefined, resultIdMap: Map<string, string>): JsonEvent;
+export declare function buildOnErrorEvent(error: TestError, workerInfo?: WorkerInfo): JsonEvent;
 export declare function buildOnStepBeginEvent(test: TestCase, result: TestResult, step: TestStep, resultIdMap: Map<string, string>, stepIdMap: Map<TestStep, string>): JsonEvent;
-export declare function buildOnStepEndEvent(test: TestCase, result: TestResult, step: TestStep, resultIdMap: Map<string, string>, stepIdMap: Map<TestStep, string>): JsonEvent;
+export declare function buildOnStepEndEvent(test: TestCase, result: TestResult, step: TestStep, resultIdMap: Map<string, string>, stepIdMap: Map<TestStep, string>, rootDir: string): JsonEvent;
 export declare function buildOnEndEvent(result: FullResult): JsonEvent;
 //# sourceMappingURL=lifecycle-events.d.ts.map

package/dist/reporter/lifecycle-events.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lifecycle-events.d.ts","sourceRoot":"","sources":["../../src/reporter/lifecycle-events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EACX,UAAU,EACV,KAAK,EACL,QAAQ,EACR,UAAU,EACV,QAAQ,EACT,MAAM,2BAA2B,CAAC;AAgBnC,MAAM,MAAM,SAAS,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,KAAK,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAClB,SAAS,EAAE,CAqDb;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,WAAW,EACpB,YAAY,EAAE,KAAK,EACnB,iBAAiB,EAAE,MAAM,GACxB,SAAS,CA+BX;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,SAAS,CAeX;AAED,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,UAAU,EAAE,MAAM,GACjB,SAAS,EAAE,CAqDb;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,QAAQ,EACd,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,GAC/B,SAAS,CAsBX;AAED,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,QAAQ,EACd,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,GAC/B,SAAS,CAuBX;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAW7D"}
+{"version":3,"file":"lifecycle-events.d.ts","sourceRoot":"","sources":["../../src/reporter/lifecycle-events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EACX,UAAU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,UAAU,EACV,QAAQ,EACR,UAAU,EACX,MAAM,2BAA2B,CAAC;AAiBnC,MAAM,MAAM,SAAS,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,KAAK,EACZ,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAClB,SAAS,EAAE,CAqDb;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,WAAW,EACpB,YAAY,EAAE,KAAK,EACnB,iBAAiB,EAAE,MAAM,GACxB,SAAS,CA+BX;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,SAAS,CAeX;AAED,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACd,SAAS,EAAE,CA+Db;AAED,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,QAAQ,GAAG,QAAQ,EACzB,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,IAAI,EAAE,QAAQ,GAAG,SAAS,EAC1B,MAAM,EAAE,UAAU,GAAG,SAAS,EAC9B,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,SAAS,CAgBX;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,SAAS,EAChB,UAAU,CAAC,EAAE,UAAU,GACtB,SAAS,CAcX;AAED,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,QAAQ,EACd,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,GAC/B,SAAS,CAsBX;AAED,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,QAAQ,EACd,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,EAChC,OAAO,EAAE,MAAM,GACd,SAAS,CA+BX;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAW7D"}

package/dist/reporter/lifecycle-events.js

@@ -7,6 +7,8 @@ exports.buildOnBeginEvents = buildOnBeginEvents;
 exports.buildOnProjectEvent = buildOnProjectEvent;
 exports.buildOnTestBeginEvent = buildOnTestBeginEvent;
 exports.buildOnTestEndEvents = buildOnTestEndEvents;
+exports.buildOnStdIOEvent = buildOnStdIOEvent;
+exports.buildOnErrorEvent = buildOnErrorEvent;
 exports.buildOnStepBeginEvent = buildOnStepBeginEvent;
 exports.buildOnStepEndEvent = buildOnStepEndEvent;
 exports.buildOnEndEvent = buildOnEndEvent;
@@ -108,7 +110,7 @@ function buildOnTestBeginEvent(test, result, resultIdMap) {
         },
     };
 }
-function buildOnTestEndEvents(test, result, resultIdMap, stagingDir) {
+function buildOnTestEndEvents(test, result, resultIdMap, stagingDir, rootDir) {
     const resultId = (0, ibr_utils_1.getOrCreateId)(resultIdMap, `${test.id}-${result.retry}`);
     const events = [];
     // onAttach for each attachment (matching Playwright's blob format)
@@ -131,6 +133,11 @@ function buildOnTestEndEvents(test, result, resultIdMap, stagingDir) {
                         name: attachment.name,
                         contentType: attachment.contentType,
                         path: attachmentPath,
+                        // Body-based attachments travel inline, mirroring
+                        // TeleReporterEmitter._serializeAttachments.
+                        base64: attachment.body
+                            ? attachment.body.toString("base64")
+                            : undefined,
                     },
                 ],
             },
@@ -143,7 +150,7 @@ function buildOnTestEndEvents(test, result, resultIdMap, stagingDir) {
                 testId: test.id,
                 expectedStatus: test.expectedStatus,
                 timeout: test.timeout,
-                annotations: test.annotations,
+                annotations: (0, ibr_utils_1.relativeAnnotationLocations)(test.annotations, rootDir),
             },
             result: {
                 id: resultId,
@@ -155,11 +162,47 @@ function buildOnTestEndEvents(test, result, resultIdMap, stagingDir) {
                     location: e.location,
                     snippet: e.snippet,
                 })),
+                // Blob v2 carries annotations on the result; the merge reads them
+                // from here (result-level), not from the test payload above.
+                annotations: result.annotations?.length
+                    ? (0, ibr_utils_1.relativeAnnotationLocations)(result.annotations, rootDir)
+                    : undefined,
             },
         },
     });
     return events;
 }
+function buildOnStdIOEvent(type, chunk, test, result, resultIdMap) {
+    const isBase64 = typeof chunk !== "string";
+    const data = isBase64 ? chunk.toString("base64") : chunk;
+    return {
+        method: "onStdIO",
+        params: {
+            testId: test?.id,
+            resultId: test && result
+                ? (0, ibr_utils_1.getOrCreateId)(resultIdMap, `${test.id}-${result.retry}`)
+                : undefined,
+            type,
+            data,
+            isBase64,
+        },
+    };
+}
+function buildOnErrorEvent(error, workerInfo) {
+    return {
+        method: "onError",
+        params: {
+            error,
+            workerInfo: workerInfo
+                ? {
+                    workerIndex: workerInfo.workerIndex,
+                    parallelIndex: workerInfo.parallelIndex,
+                    projectName: workerInfo.project.name,
+                }
+                : undefined,
+        },
+    };
+}
 function buildOnStepBeginEvent(test, result, step, resultIdMap, stepIdMap) {
     const resultId = (0, ibr_utils_1.getOrCreateId)(resultIdMap, `${test.id}-${result.retry}`);
     const stepId = (0, ibr_utils_1.getOrCreateStepId)(stepIdMap, step);
@@ -182,7 +225,7 @@ function buildOnStepBeginEvent(test, result, step, resultIdMap, stepIdMap) {
         },
     };
 }
-function buildOnStepEndEvent(test, result, step, resultIdMap, stepIdMap) {
+function buildOnStepEndEvent(test, result, step, resultIdMap, stepIdMap, rootDir) {
     const resultId = (0, ibr_utils_1.getOrCreateId)(resultIdMap, `${test.id}-${result.retry}`);
     const stepId = (0, ibr_utils_1.getOrCreateStepId)(stepIdMap, step);
     return {
@@ -201,6 +244,14 @@ function buildOnStepEndEvent(test, result, step, resultIdMap, stepIdMap) {
                         snippet: step.error.snippet,
                     }
                     : undefined,
+                // Indices into the result's attachments, matching
+                // TeleReporterEmitter._serializeStepEnd.
+                attachments: step.attachments.length
+                    ? step.attachments.map((a) => result.attachments.indexOf(a))
+                    : undefined,
+                annotations: step.annotations.length
+                    ? (0, ibr_utils_1.relativeAnnotationLocations)(step.annotations, rootDir)
+                    : undefined,
             },
         },
     };

package/dist/reporter/parity-utils.d.ts

@@ -0,0 +1,63 @@
+import type { BlobEvent } from "./harness";
+export interface NormalizedAttachment {
+    name: string;
+    contentType: string;
+    /**
+     * Whether the merged report still points at real content (path or inline
+     * body). Compared only where a scenario opts in: the empirical pipeline
+     * resolves attachment bytes through _empirical_urls.json, so name+type is
+     * the baseline parity contract.
+     */
+    hasContent: boolean;
+}
+export interface NormalizedResult {
+    retry: number;
+    status: string;
+    errorMessages: string[];
+    stdout: string[];
+    stderr: string[];
+    attachments: NormalizedAttachment[];
+}
+export interface NormalizedTest {
+    projectName: string;
+    expectedStatus: string;
+    status: string;
+    annotations: {
+        type: string;
+        description?: string;
+    }[];
+    results: NormalizedResult[];
+}
+export interface NormalizedSpec {
+    title: string;
+    file: string;
+    ok: boolean;
+    tests: NormalizedTest[];
+}
+export interface NormalizedSuite {
+    title: string;
+    file: string;
+    specs: NormalizedSpec[];
+    suites: NormalizedSuite[];
+}
+export interface NormalizedReport {
+    suites: NormalizedSuite[];
+    errorMessages: string[];
+    stats: Record<string, unknown>;
+}
+export interface NormalizeOptions {
+    /** Absolute fixture dir, replaced with a stable token in messages. */
+    cwd: string;
+    /** Include attachment `hasContent` in the comparison (see above). */
+    compareAttachmentContent?: boolean;
+}
+export declare function normalizeJsonReport(report: Record<string, unknown>, opts: NormalizeOptions): NormalizedReport;
+/**
+ * Per-result step signatures from a raw blob event stream, keyed by
+ * `testId:retry` (stable across both reporters — they see the same run).
+ * The merged JSON report drops steps, so step parity is checked here.
+ */
+export declare function stepSignatures(events: BlobEvent[]): Map<string, string[]>;
+/** Event method names present in `reference` but absent from `candidate`. */
+export declare function missingEventMethods(reference: BlobEvent[], candidate: BlobEvent[]): string[];
+//# sourceMappingURL=parity-utils.d.ts.map

package/dist/reporter/parity-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parity-utils.d.ts","sourceRoot":"","sources":["../../src/reporter/parity-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAiD3C,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,oBAAoB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAC;IACZ,qEAAqE;IACrE,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACpC;AAwED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,IAAI,EAAE,gBAAgB,GACrB,gBAAgB,CAclB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA+BzE;AAED,6EAA6E;AAC7E,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,SAAS,EAAE,EACtB,SAAS,EAAE,SAAS,EAAE,GACrB,MAAM,EAAE,CASV"}

package/dist/reporter/parity-utils.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeJsonReport = normalizeJsonReport;
+exports.stepSignatures = stepSignatures;
+exports.missingEventMethods = missingEventMethods;
+const ANSI_RE = /\u001b\[[0-9;]*m/g;
+function normalizeText(text, cwd) {
+    return text.replace(ANSI_RE, "").split(cwd).join("<cwd>").trimEnd();
+}
+function normalizeStdio(entries, cwd) {
+    return (entries ?? []).map((entry) => typeof entry === "string"
+        ? normalizeText(entry, cwd)
+        : entry.text !== undefined
+            ? normalizeText(entry.text, cwd)
+            : `<buffer:${entry.buffer?.length ?? 0}>`);
+}
+function normalizeResult(result, opts) {
+    return {
+        retry: result.retry,
+        status: result.status,
+        errorMessages: (result.errors ?? []).map((e) => normalizeText(e.message ?? "", opts.cwd)),
+        stdout: normalizeStdio(result.stdout, opts.cwd),
+        stderr: normalizeStdio(result.stderr, opts.cwd),
+        attachments: (result.attachments ?? []).map((a) => ({
+            name: a.name,
+            contentType: a.contentType,
+            hasContent: opts.compareAttachmentContent
+                ? Boolean(a.path || a.body)
+                : true,
+        })),
+    };
+}
+function normalizeSuite(suite, opts) {
+    return {
+        title: suite.title,
+        file: suite.file,
+        specs: (suite.specs ?? [])
+            .map((spec) => ({
+            title: spec.title,
+            file: spec.file,
+            ok: spec.ok,
+            tests: (spec.tests ?? []).map((test) => ({
+                projectName: test.projectName,
+                expectedStatus: test.expectedStatus,
+                status: test.status,
+                annotations: test.annotations ?? [],
+                results: (test.results ?? [])
+                    .map((r) => normalizeResult(r, opts))
+                    .sort((a, b) => a.retry - b.retry),
+            })),
+        }))
+            .sort((a, b) => a.title.localeCompare(b.title)),
+        suites: (suite.suites ?? [])
+            .map((s) => normalizeSuite(s, opts))
+            .sort((a, b) => a.title.localeCompare(b.title)),
+    };
+}
+function normalizeJsonReport(report, opts) {
+    const typed = report;
+    const stats = { ...(typed.stats ?? {}) };
+    delete stats.startTime;
+    delete stats.duration;
+    return {
+        suites: (typed.suites ?? [])
+            .map((s) => normalizeSuite(s, opts))
+            .sort((a, b) => (a.file + a.title).localeCompare(b.file + b.title)),
+        errorMessages: (typed.errors ?? []).map((e) => normalizeText(e.message ?? "", opts.cwd)),
+        stats,
+    };
+}
+/**
+ * Per-result step signatures from a raw blob event stream, keyed by
+ * `testId:retry` (stable across both reporters — they see the same run).
+ * The merged JSON report drops steps, so step parity is checked here.
+ */
+function stepSignatures(events) {
+    const resultKey = new Map(); // resultId -> testId:retry
+    for (const event of events) {
+        if (event.method !== "onTestBegin") {
+            continue;
+        }
+        const params = event.params;
+        resultKey.set(params.result.id, `${params.testId}:${params.result.retry}`);
+    }
+    const signatures = new Map();
+    for (const event of events) {
+        if (event.method !== "onStepBegin") {
+            continue;
+        }
+        const params = event.params;
+        const key = resultKey.get(params.resultId);
+        if (!key) {
+            continue;
+        }
+        const list = signatures.get(key) ?? [];
+        list.push(`${params.step.category}:${params.step.title}`);
+        signatures.set(key, list);
+    }
+    return signatures;
+}
+/** Event method names present in `reference` but absent from `candidate`. */
+function missingEventMethods(reference, candidate) {
+    const candidateMethods = new Set(candidate.map((e) => e.method));
+    return [
+        ...new Set(reference
+            .map((e) => e.method)
+            .filter((method) => !candidateMethods.has(method))),
+    ].sort();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.58.1",
+  "version": "0.58.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -30,16 +30,16 @@
   },
   "author": "Empirical Team <hey@empirical.run>",
   "devDependencies": {
-    "@playwright/test": "1.60.0",
+    "@playwright/test": "1.62.1",
     "@types/async-retry": "^1.4.8",
     "@types/authenticator": "^1.1.4",
     "@types/babel__code-frame": "^7.0.6",
     "@types/console-log-level": "^1.4.5",
     "@types/mime": "3.0.0",
-    "@types/node": "^20.14.9",
+    "@types/node": "^22",
     "@types/serve-handler": "^6.1.4",
     "openapi-typescript": "7.6.1",
-    "playwright-core": "1.60.0",
+    "playwright-core": "1.62.1",
     "serve-handler": "^6.1.6",
     "@empiricalrun/shared-types": "0.15.3"
   },
@@ -54,8 +54,8 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/cua": "^0.4.1",
     "@empiricalrun/llm": "^0.27.0",
+    "@empiricalrun/cua": "^0.4.1",
     "@empiricalrun/r2-uploader": "^0.10.1",
     "@empiricalrun/reporter": "^0.29.1"
   },

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}
+{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/parity-utils.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}