@empiricalrun/playwright-utils 0.14.17 → 0.14.19

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/playwright-utils
 
+## 0.14.19
+
+### Patch Changes
+
+- Updated dependencies [e547617]
+  - @empiricalrun/llm@0.9.12
+
+## 0.14.18
+
+### Patch Changes
+
+- 0e75cfc: feat: custom reporter checks all attachments are processed after test run
+
 ## 0.14.17
 
 ### Patch Changes

package/dist/reporter/custom.d.ts

@@ -42,6 +42,7 @@ declare class HtmlReporter implements ReporterV2 {
     private uploadExecutionQueue;
     private uploadWaitingQueue;
     private uploadMaxQueueSize;
+    private haveSeenAttachments;
     private processQueue;
     private waitForAllTasksToFinish;
     private _buildResult;
@@ -55,6 +56,7 @@ declare class HtmlReporter implements ReporterV2 {
     onTestBegin(): void;
     version(): "v2";
     onTestEnd(test: TestCasePublic, result: TestResultPublic): void;
+    private processTestAttachment;
     onConfigure(config: FullConfig): void;
     onBegin(suite: Suite): void;
     _resolveOptions(): {

package/dist/reporter/custom.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/reporter/custom.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EAEV,KAAK,EACL,QAAQ,IAAI,cAAc,EAC1B,SAAS,EACT,UAAU,IAAI,gBAAgB,EAE/B,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAML,UAAU,EAMX,MAAM,2BAA2B,CAAC;AAiBnC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAiC/C,QAAA,MAAM,iBAAiB,UAAoC,CAAC;AAC5D,KAAK,oBAAoB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAM/D,KAAK,mBAAmB,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,oBAAoB,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AA6BF,cAAM,YAAa,YAAW,UAAU;IACtC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,mBAAmB,CAAU;IACrC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,oBAAoB,CAAiC;IAC7D,OAAO,CAAC,kBAAkB,CAAyC;IACnE,OAAO,CAAC,kBAAkB,CAEpB;IAEN,OAAO,CAAC,YAAY;YA4BN,uBAAuB;IAQrC,OAAO,CAAC,YAAY,CAEN;IACd,OAAO,CAAC,eAAe,CAAmB;gBAE9B,OAAO,EAAE,mBAAmB;IAIxC,aAAa;IAIb,QAAQ;IAER,QAAQ;IAER,WAAW;IAEX,SAAS;IAET,WAAW;IAEX,OAAO,IAAI,IAAI;IAIf,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB;IA0DxD,WAAW,CAAC,MAAM,EAAE,UAAU;IAI9B,OAAO,CAAC,KAAK,EAAE,KAAK;IAYpB,eAAe,IAAI;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,oBAAoB,CAAC;QAC3B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B;IAsBD,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IASxD,OAAO,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI;IAIzB,KAAK,CAAC,MAAM,EAAE,UAAU;IAoIxB,MAAM;CA8Bb;AAkCD,wBAAsB,cAAc,CAClC,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,IAAI,GAAE,MAAoB,EAC1B,IAAI,CAAC,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,iBAqBhB;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAsBhE;AAqiBD,eAAe,YAAY,CAAC"}
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/reporter/custom.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EAMV,KAAK,EACL,QAAQ,IAAI,cAAc,EAC1B,SAAS,EACT,UAAU,IAAI,gBAAgB,EAE/B,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAML,UAAU,EAMX,MAAM,2BAA2B,CAAC;AAiBnC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAiC/C,QAAA,MAAM,iBAAiB,UAAoC,CAAC;AAC5D,KAAK,oBAAoB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAM/D,KAAK,mBAAmB,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,oBAAoB,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AA6BF,cAAM,YAAa,YAAW,UAAU;IACtC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,mBAAmB,CAAU;IACrC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,oBAAoB,CAAiC;IAC7D,OAAO,CAAC,kBAAkB,CAAyC;IACnE,OAAO,CAAC,kBAAkB,CAEpB;IACN,OAAO,CAAC,mBAAmB,CAAqB;IAEhD,OAAO,CAAC,YAAY;YA4BN,uBAAuB;IAQrC,OAAO,CAAC,YAAY,CAEN;IACd,OAAO,CAAC,eAAe,CAAmB;gBAE9B,OAAO,EAAE,mBAAmB;IAIxC,aAAa;IAIb,QAAQ;IAER,QAAQ;IAER,WAAW;IAEX,SAAS;IAET,WAAW;IAEX,OAAO,IAAI,IAAI;IAIf,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB;IAUxD,OAAO,CAAC,qBAAqB;IAoD7B,WAAW,CAAC,MAAM,EAAE,UAAU;IAI9B,OAAO,CAAC,KAAK,EAAE,KAAK;IAYpB,eAAe,IAAI;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,oBAAoB,CAAC;QAC3B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;KAC1B;IAsBD,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IASxD,OAAO,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI;IAIzB,KAAK,CAAC,MAAM,EAAE,UAAU;IA+IxB,MAAM;CA8Bb;AAwED,wBAAsB,cAAc,CAClC,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,IAAI,GAAE,MAAoB,EAC1B,IAAI,CAAC,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,iBAqBhB;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAsBhE;AAqiBD,eAAe,YAAY,CAAC"}

package/dist/reporter/custom.js

@@ -75,6 +75,7 @@ class HtmlReporter {
     uploadMaxQueueSize = process.env.UPLOAD_MAX_QUEUE_SIZE
         ? parseInt(process.env.UPLOAD_MAX_QUEUE_SIZE)
         : 2;
+    haveSeenAttachments = new Set();
     processQueue() {
         try {
             while (this.uploadExecutionQueue.length <= this.uploadMaxQueueSize &&
@@ -125,55 +126,60 @@ class HtmlReporter {
     onTestEnd(test, result) {
         try {
             result.attachments.forEach(async (attachment) => {
-                if (!attachment.path)
-                    return;
-                const folder = path_1.default.relative(this._outputFolder + "/data", attachment.path);
-                const folderName = folder.split("/")[0];
-                // on test gen run, we don't have PROJECT_NAME or TEST_RUN_GITHUB_ACTION_ID so we skip uploading since we anyways handle uploads for that, this is a stopgap
-                if (!process.env.PROJECT_NAME ||
-                    !process.env.TEST_RUN_GITHUB_ACTION_ID) {
-                    return;
-                }
-                try {
-                    const uploadTask = async () => (0, r2_uploader_1.uploadDirectory)({
-                        fileList: [attachment.path],
-                        sourceDir: this._outputFolder + "/data/" + folderName,
-                        destinationDir: process.env.PROJECT_NAME +
-                            "/" +
-                            process.env.TEST_RUN_GITHUB_ACTION_ID +
-                            "/data/" +
-                            folderName,
-                        uploadBucket: "test-report",
-                    });
-                    if (this.uploadExecutionQueue.length <= this.uploadMaxQueueSize) {
-                        const promise = uploadTask()
-                            .catch((e) => {
-                            logger_1.logger.error("Upload failed for", attachment.path, e);
-                            // this.retryTask(uploadTask); // Retry logic
-                        })
-                            .finally(() => {
-                            // this is a list of promises which we dont want to await right now
-                            // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                            this.uploadExecutionQueue = this.uploadExecutionQueue.filter((p) => p !== promise);
-                            this.processQueue(); // Keep processing queue
-                        });
-                        this.uploadExecutionQueue.push(promise);
-                    }
-                    else {
-                        logger_1.logger.debug("Queuing upload task ", attachment.path);
-                        this.uploadWaitingQueue.push(uploadTask);
-                    }
-                }
-                catch (e) {
-                    logger_1.logger.error("Error while uploading attachment", e);
-                    // tests shouldn't stop no matter whatever error we get
-                }
+                return this.processTestAttachment(attachment);
             });
         }
         catch (e) {
             logger_1.logger.error("Error while processing attachments for test", e);
         }
     }
+    processTestAttachment(attachment) {
+        if (!attachment.path)
+            return;
+        if (this.haveSeenAttachments.has(attachment.path))
+            return;
+        this.haveSeenAttachments.add(attachment.path);
+        const folder = path_1.default.relative(this._outputFolder + "/data", attachment.path);
+        const folderName = folder.split("/")[0];
+        // on test gen run, we don't have PROJECT_NAME or TEST_RUN_GITHUB_ACTION_ID so we skip uploading since we anyways handle uploads for that, this is a stopgap
+        if (!process.env.PROJECT_NAME || !process.env.TEST_RUN_GITHUB_ACTION_ID) {
+            return;
+        }
+        try {
+            const uploadTask = async () => (0, r2_uploader_1.uploadDirectory)({
+                fileList: [attachment.path],
+                sourceDir: this._outputFolder + "/data/" + folderName,
+                destinationDir: process.env.PROJECT_NAME +
+                    "/" +
+                    process.env.TEST_RUN_GITHUB_ACTION_ID +
+                    "/data/" +
+                    folderName,
+                uploadBucket: "test-report",
+            });
+            if (this.uploadExecutionQueue.length <= this.uploadMaxQueueSize) {
+                const promise = uploadTask()
+                    .catch((e) => {
+                    logger_1.logger.error("Upload failed for", attachment.path, e);
+                    // this.retryTask(uploadTask); // Retry logic
+                })
+                    .finally(() => {
+                    // this is a list of promises which we dont want to await right now
+                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
+                    this.uploadExecutionQueue = this.uploadExecutionQueue.filter((p) => p !== promise);
+                    this.processQueue(); // Keep processing queue
+                });
+                this.uploadExecutionQueue.push(promise);
+            }
+            else {
+                logger_1.logger.debug("Queuing upload task ", attachment.path);
+                this.uploadWaitingQueue.push(uploadTask);
+            }
+        }
+        catch (e) {
+            logger_1.logger.error("Error while uploading attachment", e);
+            // tests shouldn't stop no matter whatever error we get
+        }
+    }
     onConfigure(config) {
         this.config = config;
     }
@@ -231,6 +237,15 @@ class HtmlReporter {
         try {
             logger_1.logger.debug("uploading path", this._outputFolder);
             logger_1.logger.debug("file list", listFilesRecursively(this._outputFolder));
+            const summaryJsonPath = path_1.default.join(this._outputFolder, "summary.json");
+            if (fs_1.default.existsSync(summaryJsonPath)) {
+                logger_1.logger.debug("reading json to ensure all attachments are processed");
+                const summaryJson = JSON.parse(fs_1.default.readFileSync(summaryJsonPath, "utf-8"));
+                const attachments = getAttachmentsFromJSONReport(summaryJson);
+                attachments.forEach(async (attachment) => {
+                    return this.processTestAttachment(attachment);
+                });
+            }
             const tracePath = this._outputFolder + "/trace";
             const traceExists = fs_1.default.existsSync(tracePath);
             logger_1.logger.debug("trace exists:", traceExists);
@@ -333,6 +348,42 @@ class HtmlReporter {
         }
     }
 }
+function getAttachmentsFromJSONReport(report) {
+    const { suites } = report;
+    return getAttachmentsFromSuites(suites);
+}
+function getAttachmentsFromSuites(suites) {
+    if (!suites)
+        return [];
+    if (!suites.length)
+        return [];
+    let allAttachments = [];
+    suites.forEach((currentSuite) => {
+        const { specs } = currentSuite;
+        const { suites } = currentSuite;
+        const attachmentsFromSuites = getAttachmentsFromSuites(suites);
+        const attachmentsFromSpecs = getAttachmentsFromSpecs(specs);
+        allAttachments = allAttachments.concat(attachmentsFromSuites, attachmentsFromSpecs);
+    });
+    return allAttachments;
+}
+function getAttachmentsFromSpecs(specs) {
+    if (!specs)
+        return [];
+    if (!specs.length)
+        return [];
+    let allAttachments = [];
+    specs.forEach((spec) => {
+        spec.tests.forEach((test) => {
+            const { results } = test;
+            results.forEach((result) => {
+                const { attachments } = result;
+                allAttachments = allAttachments.concat(attachments);
+            });
+        });
+    });
+    return allAttachments;
+}
 function reportFolderFromEnv() {
     // Note: PLAYWRIGHT_HTML_REPORT is for backwards compatibility.
     const envValue = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.14.17",
+  "version": "0.14.19",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -40,7 +40,7 @@
     "playwright-core": "1.47.1",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
-    "@empiricalrun/llm": "^0.9.11",
+    "@empiricalrun/llm": "^0.9.12",
     "@empiricalrun/r2-uploader": "^0.3.6"
   },
   "scripts": {