@empiricalrun/playwright-utils 0.46.1 → 0.46.3

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/playwright-utils
 
+## 0.46.3
+
+### Patch Changes
+
+- 3a315aa: fix: pass->skip->fail retries were treated as failed test
+- 010eda2: fix: upload incremental report only when reporter has begun
+- Updated dependencies [3a315aa]
+  - @empiricalrun/reporter@0.28.1
+
+## 0.46.2
+
+### Patch Changes
+
+- 364eb85: fix: skipped tests have 1 retry. persist them
+
 ## 0.46.1
 
 ### Patch Changes

package/dist/reporter/ibr-utils.d.ts

@@ -40,6 +40,7 @@ export declare function serializeSuite(suite: Suite, rootDir: string): JsonSuite
  * Returns null if the suite has no completed test descendants.
  */
 export declare function filterSuiteByCompletedTests(suite: JsonSuite, completedTestIds: Set<string>): JsonSuite | null;
+export declare function hasValidReport(reportLines: string[]): boolean;
 export declare function buildUrlsJson(attachmentUrlMap: Map<string, string>): Record<string, string>;
 export declare function isLocalTesting(): boolean;
 export declare function embedAttachment(attachmentPath: string | undefined, resultId: string, stagingDir: string): string | null;

package/dist/reporter/ibr-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ibr-utils.d.ts","sourceRoot":"","sources":["../../src/reporter/ibr-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAO3E,MAAM,MAAM,WAAW,GAAG;IACxB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC,CAAC;AAIF,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAChD,WAAW,EAAE,CAOf;AAED,wBAAgB,UAAU,IAAI,MAAM,CAEnC;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAO3E;AAED,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,EAC1B,IAAI,EAAE,QAAQ,GACb,MAAM,CAOR;AAID,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAIF,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD,CAAC;AAIF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,CAAC,YAAY,GAAG,SAAS,CAAC,EAAE,CAAC;CACvC,CAAC;AAIF,wBAAgB,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,YAAY,CAc3E;AAID,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAsBvE;AAMD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,SAAS,EAChB,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC5B,SAAS,GAAG,IAAI,CAmBlB;AAED,wBAAgB,aAAa,CAC3B,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAMxB;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED,wBAAgB,eAAe,CAC7B,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB,MAAM,GAAG,IAAI,CAkBf"}
+{"version":3,"file":"ibr-utils.d.ts","sourceRoot":"","sources":["../../src/reporter/ibr-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAO3E,MAAM,MAAM,WAAW,GAAG;IACxB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC,CAAC;AAIF,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAChD,WAAW,EAAE,CAOf;AAED,wBAAgB,UAAU,IAAI,MAAM,CAEnC;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAO3E;AAED,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,EAC1B,IAAI,EAAE,QAAQ,GACb,MAAM,CAOR;AAID,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAIF,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD,CAAC;AAIF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,CAAC,YAAY,GAAG,SAAS,CAAC,EAAE,CAAC;CACvC,CAAC;AAIF,wBAAgB,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,YAAY,CAc3E;AAID,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAsBvE;AAMD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,SAAS,EAChB,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC5B,SAAS,GAAG,IAAI,CAmBlB;AAED,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAK7D;AAED,wBAAgB,aAAa,CAC3B,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAMxB;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED,wBAAgB,eAAe,CAC7B,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB,MAAM,GAAG,IAAI,CAkBf"}

package/dist/reporter/ibr-utils.js

@@ -10,6 +10,7 @@ exports.getOrCreateStepId = getOrCreateStepId;
 exports.serializeTest = serializeTest;
 exports.serializeSuite = serializeSuite;
 exports.filterSuiteByCompletedTests = filterSuiteByCompletedTests;
+exports.hasValidReport = hasValidReport;
 exports.buildUrlsJson = buildUrlsJson;
 exports.isLocalTesting = isLocalTesting;
 exports.embedAttachment = embedAttachment;
@@ -109,6 +110,12 @@ function filterSuiteByCompletedTests(suite, completedTestIds) {
         return null;
     return { ...suite, entries: filteredEntries };
 }
+function hasValidReport(reportLines) {
+    return reportLines.some((line) => {
+        const parsed = JSON.parse(line);
+        return parsed.method === "onBegin";
+    });
+}
 function buildUrlsJson(attachmentUrlMap) {
     const urlMap = {};
     for (const [localPath, url] of attachmentUrlMap) {

package/dist/reporter/incremental-blob-reporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-blob-reporter.d.ts","sourceRoot":"","sources":["../../src/reporter/incremental-blob-reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,EACR,UAAU,EACV,QAAQ,EACT,MAAM,2BAA2B,CAAC;AAsBnC;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEnD,cAAM,uBAAwB,YAAW,QAAQ;IAC/C,OAAO,CAAC,kBAAkB,CAAyB;IACnD,OAAO,CAAC,WAAW,CAIjB;IACF,OAAO,CAAC,UAAU,CAGhB;IACF,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,iBAAiB,CAA+B;IACxD,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,iBAAiB,CAA0B;IACnD,OAAO,CAAC,mBAAmB,CAA0B;IACrD,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,YAAY,CAGN;IACd,OAAO,CAAC,UAAU,CAAsB;;IAexC,OAAO,CAAC,cAAc,CAA6B;IAEnD,OAAO,CAAC,mBAAmB;IA0B3B,OAAO,CAAC,oBAAoB;YAOd,eAAe;IAiC7B,OAAO,CAAC,2BAA2B;IAyBnC,OAAO,CAAC,qBAAqB;IAiB7B,OAAO,CAAC,8BAA8B;IAStC,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,KAAK,QAAQ,GAKnB;IAED,OAAO,KAAK,UAAU,GAErB;IAED,OAAO,KAAK,aAAa,GAExB;IAED,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,aAAa;YAUP,SAAS;IAwBvB,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAgB/C,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAIrD,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAuBnD,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,GAAG,IAAI;IAYrE,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,GAAG,IAAI;IAY7D,KAAK,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB9C;;OAEG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAI5D,aAAa,IAAI,OAAO;CAGzB;AAID,wBAAgB,0BAA0B,IAAI,uBAAuB,GAAG,IAAI,CAE3E;AAED,wBAAgB,kCAAkC,CAChD,QAAQ,EAAE,uBAAuB,GAChC,IAAI,CAEN;AAED,eAAe,uBAAuB,CAAC"}
+{"version":3,"file":"incremental-blob-reporter.d.ts","sourceRoot":"","sources":["../../src/reporter/incremental-blob-reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,QAAQ,EACR,KAAK,EACL,QAAQ,EACR,UAAU,EACV,QAAQ,EACT,MAAM,2BAA2B,CAAC;AAuBnC;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEnD,cAAM,uBAAwB,YAAW,QAAQ;IAC/C,OAAO,CAAC,kBAAkB,CAAyB;IACnD,OAAO,CAAC,WAAW,CAIjB;IACF,OAAO,CAAC,UAAU,CAGhB;IACF,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,iBAAiB,CAA+B;IACxD,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,iBAAiB,CAA0B;IACnD,OAAO,CAAC,mBAAmB,CAA0B;IACrD,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,YAAY,CAGN;IACd,OAAO,CAAC,UAAU,CAAsB;;IAexC,OAAO,CAAC,cAAc,CAA6B;IAEnD,OAAO,CAAC,mBAAmB;IA0B3B,OAAO,CAAC,oBAAoB;YAOd,eAAe;IAwC7B,OAAO,CAAC,2BAA2B;IAyBnC,OAAO,CAAC,qBAAqB;IAY7B,OAAO,CAAC,8BAA8B;IAStC,OAAO,CAAC,mBAAmB;IAkB3B,OAAO,KAAK,QAAQ,GAKnB;IAED,OAAO,KAAK,UAAU,GAErB;IAED,OAAO,KAAK,aAAa,GAExB;IAED,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,aAAa;YAUP,SAAS;IAwBvB,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAgB/C,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAIrD,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IAyBnD,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,GAAG,IAAI;IAYrE,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,GAAG,IAAI;IAY7D,KAAK,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB9C;;OAEG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAI5D,aAAa,IAAI,OAAO;CAGzB;AAID,wBAAgB,0BAA0B,IAAI,uBAAuB,GAAG,IAAI,CAE3E;AAED,wBAAgB,kCAAkC,CAChD,QAAQ,EAAE,uBAAuB,GAChC,IAAI,CAEN;AAED,eAAe,uBAAuB,CAAC"}

package/dist/reporter/incremental-blob-reporter.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getIncrementalBlobReporter = getIncrementalBlobReporter;
 exports.setIncrementalBlobReporterInstance = setIncrementalBlobReporterInstance;
 const zip_1 = require("@empiricalrun/r2-uploader/zip");
+const reporter_1 = require("@empiricalrun/reporter");
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const ibr_utils_1 = require("./ibr-utils");
@@ -28,7 +29,7 @@ class IncrementalBlobReporter {
     _completedTestIds = new Set();
     _completedResultIds = new Set();
     _testRetries = new Map(); // testId → configured retries
-    _testResults = new Map(); // testId → retryIndex → {retryId, retryStatus}
+    _testResults = new Map(); // testId → retryIndex → {resultId, status, expectedStatus}
     _startTime = Date.now();
     constructor() {
         if (process.env.SHARD_INDEX && process.env.TOTAL_SHARDS) {
@@ -55,7 +56,7 @@ class IncrementalBlobReporter {
                 process.exit(0);
             })
                 .catch((err) => {
-                console.error("[IncrementalBlobReporter] Flush/upload failed on SIGINT:", err);
+                console.error("[IncrementalBlobReporterFailure] Flush/upload failed on SIGINT:", err);
             })
                 .finally(() => {
                 this._removeSignalHandler();
@@ -71,6 +72,10 @@ class IncrementalBlobReporter {
     }
     async _flushAndUpload() {
         (0, reporter_state_1.setFinalized)();
+        if (!(0, ibr_utils_1.hasValidReport)(this._reportLines)) {
+            console.warn("[IncrementalBlobReporterFailure] No onBegin received, skipping upload (nothing to merge)");
+            return;
+        }
         // Wait for pending attachment uploads FIRST so URLs are available
         if (this._uploader) {
             await this._uploader.waitForUploads();
@@ -78,7 +83,7 @@ class IncrementalBlobReporter {
         this._finalizeReportForInterrupt();
         await this._writeZip();
         if (!this._uploader) {
-            console.log("[IncrementalBlobReporter] No uploader available, skipping upload");
+            console.warn("[IncrementalBlobReporterFailure] No uploader available, skipping upload");
             return;
         }
         await this._uploader.uploadFile(this._zipPath, `blobs/incremental-report-${this._shardIndex}.zip`);
@@ -111,10 +116,8 @@ class IncrementalBlobReporter {
     }
     _computeCompletedSets() {
         for (const [testId, results] of this._testResults) {
-            const configuredRetries = this._testRetries.get(testId);
-            const hasPassedRetry = [...results.values()].some((r) => r.status === "passed");
-            const allRetriesCompleted = results.size === configuredRetries + 1;
-            if (hasPassedRetry || allRetriesCompleted) {
+            const maxRetries = this._testRetries.get(testId);
+            if ((0, reporter_1.areRetriesComplete)(maxRetries, results)) {
                 this._completedTestIds.add(testId);
                 for (const { resultId } of results.values()) {
                     this._completedResultIds.add(resultId);
@@ -205,9 +208,11 @@ class IncrementalBlobReporter {
             }
             const resultId = this._resultIdMap.get(`${test.id}-${result.retry}`);
             if (resultId) {
-                this._testResults
-                    .get(test.id)
-                    .set(result.retry, { resultId, status: result.status });
+                this._testResults.get(test.id).set(result.retry, {
+                    resultId,
+                    status: result.status,
+                    expectedStatus: test.expectedStatus,
+                });
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.46.1",
+  "version": "0.46.3",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -43,11 +43,11 @@
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
+    "@empiricalrun/cua": "^0.3.0",
     "@empiricalrun/dashboard-client": "^0.2.0",
     "@empiricalrun/llm": "^0.26.0",
     "@empiricalrun/r2-uploader": "^0.9.1",
-    "@empiricalrun/reporter": "^0.28.0",
-    "@empiricalrun/cua": "^0.3.0"
+    "@empiricalrun/reporter": "^0.28.1"
   },
   "scripts": {
     "dev": "tsc --build --watch",