@empiricalrun/playwright-utils 0.26.21 → 0.27.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @empiricalrun/playwright-utils
 
+## 0.27.0
+
+### Minor Changes
+
+- 6390198: feat: add EmpiricalReporter for realtime upload of test artifacts & test results
+
+### Patch Changes
+
+- Updated dependencies [33abadf]
+- Updated dependencies [1300a80]
+- Updated dependencies [985d721]
+- Updated dependencies [576870a]
+- Updated dependencies [e94a2da]
+- Updated dependencies [41c266d]
+- Updated dependencies [dfb2007]
+- Updated dependencies [6a19421]
+- Updated dependencies [1a46013]
+- Updated dependencies [805f35f]
+- Updated dependencies [0a9ec78]
+- Updated dependencies [c430cc0]
+  - @empiricalrun/test-gen@0.66.0
+  - @empiricalrun/llm@0.18.1
+
 ## 0.26.21
 
 ### Patch Changes

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAczD,wBAAgB,gBAAgB,IAAI,MAAM,CAazC;AAED,eAAO,MAAM,UAAU,EAAE,oBA6BxB,CAAC;AAEF,eAAO,MAAM,OAAO,EAAE,iBAgBrB,CAAC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAI7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAczD,wBAAgB,gBAAgB,IAAI,MAAM,CAazC;AA0CD,eAAO,MAAM,UAAU,EAAE,oBAyBxB,CAAC;AAEF,eAAO,MAAM,OAAO,EAAE,iBAgBrB,CAAC"}

package/dist/config.js

@@ -5,6 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.devices = exports.baseConfig = void 0;
 exports.chromeStablePath = chromeStablePath;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const deviceDescriptorsSource_json_1 = __importDefault(require("./devices/v1.47.1/deviceDescriptorsSource.json"));
 function getPlatformOS() {
     const platform = process.platform;
@@ -35,10 +37,43 @@ function chromeStablePath() {
         }
     }
 }
+function getPlaywrightVersion() {
+    try {
+        const packageJsonPath = path_1.default.join(process.cwd(), "node_modules", "@playwright", "test", "package.json");
+        const packageJson = JSON.parse(fs_1.default.readFileSync(packageJsonPath, "utf8"));
+        return packageJson.version;
+    }
+    catch (error) {
+        console.log("Package not found");
+        return "";
+    }
+}
+function isPlaywrightV147() {
+    const version = getPlaywrightVersion();
+    return version && version.includes("1.47") ? true : false;
+}
+function getReporters() {
+    if (isPlaywrightV147()) {
+        return [
+            ["list"],
+            ["json", { outputFile: "playwright-report/summary.json" }],
+            ["./node_modules/@empiricalrun/playwright-utils/dist/reporter/custom.js"],
+        ];
+    }
+    return [
+        ["list"],
+        ["json", { outputFile: "summary.json" }],
+        ["html", { open: "never" }],
+        [
+            "./node_modules/@empiricalrun/playwright-utils/dist/reporter/empirical-reporter.js",
+        ],
+    ];
+}
 exports.baseConfig = {
     testDir: "./tests",
     // outputDir is where we store playwright artifacts (during and post test run)
-    outputDir: "./playwright-report/data",
+    outputDir: isPlaywrightV147() ? "./playwright-report/data" : "./test-results",
+    reporter: getReporters(),
     fullyParallel: true,
     forbidOnly: false,
     retries: process.env.CI ? 2 : 0,
@@ -46,11 +81,6 @@ exports.baseConfig = {
     workers: undefined,
     // maxFailures counts retries as individual failures, so this is equivalent to 20 failed tests
     maxFailures: process.env.CI ? 60 : undefined,
-    reporter: [
-        ["list"], // For real-time reporting on CI terminal (vs. the default "dot" reporter)
-        ["json", { outputFile: "playwright-report/summary.json" }],
-        ["./node_modules/@empiricalrun/playwright-utils/dist/reporter/custom.js"],
-    ],
     use: {
         trace: "on",
         video: {

package/dist/overlay-tests/cache.spec.js

@@ -38,7 +38,6 @@ fixtures_1.test.afterEach(() => {
     ]);
     await page.goto(`${server.baseURL}/glass-pane.html`);
     let error;
-    let startTime = Date.now();
     // Overwriting API key to ensure we don't call OpenAI
     const originalApiKey = process.env.OPENAI_API_KEY;
     process.env.OPENAI_API_KEY = undefined;
@@ -50,12 +49,7 @@ fixtures_1.test.afterEach(() => {
     catch (e) {
         error = e;
     }
-    process.env.OPENAI_API_KEY = originalApiKey;
-    const duration = Date.now() - startTime;
     (0, fixtures_1.expect)(error).toBeDefined();
+    process.env.OPENAI_API_KEY = originalApiKey;
     (0, fixtures_1.expect)(error.message).toContain('<div class="glass-pane"></div> intercepts pointer events');
-    // Expect total time to be action > timeout * 2 (original + dismiss)
-    (0, fixtures_1.expect)(duration).toBeGreaterThan(10_000);
-    // + some buffer for failing API call
-    (0, fixtures_1.expect)(duration).toBeLessThan(14_999);
 });

package/dist/reporter/empirical-reporter.d.ts

@@ -0,0 +1,12 @@
+import type { FullResult, Reporter, TestCase, TestResult } from "@playwright/test/reporter";
+declare class EmpiricalReporter implements Reporter {
+    private _destinationDir;
+    private _uploadBucket;
+    private _currentWorkingDir;
+    constructor();
+    private processTestAttachment;
+    onTestEnd(test: TestCase, result: TestResult): void;
+    onEnd(result: FullResult): Promise<void>;
+}
+export default EmpiricalReporter;
+//# sourceMappingURL=empirical-reporter.d.ts.map

package/dist/reporter/empirical-reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"empirical-reporter.d.ts","sourceRoot":"","sources":["../../src/reporter/empirical-reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,UAAU,EACX,MAAM,2BAA2B,CAAC;AAgBnC,cAAM,iBAAkB,YAAW,QAAQ;IACzC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAyB;IAC9C,OAAO,CAAC,kBAAkB,CAAyB;;IAcnD,OAAO,CAAC,qBAAqB,CA0B3B;IAEF,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU;IAyEtC,KAAK,CAAC,MAAM,EAAE,UAAU;CA4F/B;AAED,eAAe,iBAAiB,CAAC"}

package/dist/reporter/empirical-reporter.js

@@ -0,0 +1,170 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto_1 = require("crypto");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const logger_1 = require("../logger");
+const queue_1 = require("./queue");
+const uploader_1 = require("./uploader");
+const util_1 = require("./util");
+class EmpiricalReporter {
+    _destinationDir;
+    _uploadBucket = "test-report";
+    _currentWorkingDir = process.cwd();
+    constructor() {
+        if (!process.env.PROJECT_NAME || !process.env.TEST_RUN_GITHUB_ACTION_ID) {
+            logger_1.logger.error("PROJECT_NAME and TEST_RUN_GITHUB_ACTION_ID must be set");
+            this._destinationDir = "";
+            return;
+        }
+        this._destinationDir = path_1.default.join(process.env.PROJECT_NAME, process.env.TEST_RUN_GITHUB_ACTION_ID);
+    }
+    processTestAttachment = async (attachment) => {
+        if (!attachment.path)
+            return;
+        const testResultSourceDir = path_1.default.join(this._currentWorkingDir, "test-results");
+        const exists = await (0, util_1.checkFileExistsAsync)(attachment.path);
+        if (exists) {
+            logger_1.logger.info(`[Empirical Reporter] Processing attachment: File exists`);
+        }
+        else {
+            logger_1.logger.error(`[Empirical Reporter] Attachment does not exist: ${attachment.path}`);
+            return;
+        }
+        const uploadTask = (0, uploader_1.createUploadTaskV2)({
+            sourceDir: testResultSourceDir,
+            fileList: [attachment.path],
+            destinationDir: path_1.default.join(this._destinationDir, "data"),
+            uploadBucket: this._uploadBucket,
+        });
+        return (0, queue_1.sendTaskToQueue)(uploadTask);
+    };
+    onTestEnd(test, result) {
+        console.log(`[Empirical Reporter] Finished test ${test.title}: ${result.status}`);
+        if (!process.env.TEST_RUN_GITHUB_ACTION_ID) {
+            logger_1.logger.error("[Empirical Reporter] TEST_RUN_GITHUB_ACTION_ID is not set");
+            return;
+        }
+        try {
+            const attachmentPromises = result.attachments.map((attachment) => {
+                return this.processTestAttachment(attachment);
+            });
+            const testCaseRunEventTask = async () => {
+                return Promise.all(attachmentPromises)
+                    .then((uploadedAttachments) => {
+                    logger_1.logger.debug(`[Empirical Reporter] Attachments for test ${test.title} are uploaded:`, uploadedAttachments);
+                    const { suites, projectName } = (0, util_1.suitesAndProjectForTest)(test);
+                    let params = {
+                        test,
+                        result,
+                        assetsURL: {
+                            trace: "",
+                            videos: [],
+                            errorContext: "",
+                        },
+                        suites,
+                        projectName,
+                        runId: process.env.TEST_RUN_GITHUB_ACTION_ID || "",
+                    };
+                    uploadedAttachments.forEach((attachment) => {
+                        if (attachment) {
+                            Object.entries(attachment).forEach(([key, value]) => {
+                                if (key.includes("video")) {
+                                    params.assetsURL.videos.push({
+                                        name: `test-${test.id}-slug-${(0, crypto_1.randomUUID)()}`,
+                                        url: value,
+                                    });
+                                }
+                                if (key.includes("trace")) {
+                                    params.assetsURL.trace = value;
+                                }
+                                if (key.includes("error-context")) {
+                                    params.assetsURL.errorContext = value;
+                                }
+                            });
+                        }
+                    });
+                    return (0, util_1.sendTestCaseUpdateToDashboard)(params);
+                })
+                    .catch((error) => {
+                    logger_1.logger.error(`[Empirical Reporter] Error sending test case event for: ${test.title}:`, error);
+                });
+            };
+            void (0, queue_1.sendTaskToQueue)(testCaseRunEventTask);
+            return;
+        }
+        catch (error) {
+            logger_1.logger.error(`[Empirical Reporter] Error in onTestEnd for ${test.title}:`, error);
+        }
+    }
+    async onEnd(result) {
+        logger_1.logger.info(`[Empirical Reporter] Test run completed with status: ${result.status}`);
+        const startTime = new Date().getTime();
+        logger_1.logger.info("[Empirical Reporter] Starting final report upload at: ", new Intl.DateTimeFormat("en-US", {
+            hour: "2-digit",
+            minute: "2-digit",
+            second: "2-digit",
+        }).format(startTime));
+        const jsonFilePath = path_1.default.join(this._currentWorkingDir, "summary.json");
+        const jsonExists = fs_1.default.existsSync(jsonFilePath);
+        if (jsonExists) {
+            logger_1.logger.info("[Empirical Reporter] Uploading JSON file");
+            const uploadJsonTask = (0, uploader_1.createUploadTaskV2)({
+                sourceDir: this._currentWorkingDir,
+                fileList: [jsonFilePath],
+                destinationDir: this._destinationDir,
+                uploadBucket: this._uploadBucket,
+            });
+            void (0, queue_1.sendTaskToQueue)(uploadJsonTask);
+        }
+        else {
+            logger_1.logger.error(`[Empirical Reporter] summary.json does not exist at: ${jsonFilePath}`);
+        }
+        const htmlFilePath = path_1.default.join(this._currentWorkingDir, "playwright-report/index.html");
+        const htmlExists = await (0, util_1.checkFileExistsAsync)(htmlFilePath);
+        if (htmlExists) {
+            logger_1.logger.info("[Empirical Reporter] Updating the HTML Zip");
+            await (0, util_1.updateHtmlZipFileAttachmentPaths)(jsonFilePath, htmlFilePath);
+            logger_1.logger.info("[Empirical Reporter] Uploading HTML file");
+            const uploadHtmlTask = (0, uploader_1.createUploadTaskV2)({
+                sourceDir: path_1.default.join(this._currentWorkingDir, "playwright-report"),
+                fileList: [htmlFilePath],
+                destinationDir: this._destinationDir,
+                uploadBucket: this._uploadBucket,
+            });
+            void (0, queue_1.sendTaskToQueue)(uploadHtmlTask);
+        }
+        else {
+            logger_1.logger.error(`[Empirical Reporter] playwright-report/index.html does not exist at: ${htmlFilePath}`);
+        }
+        const traceFilePath = path_1.default.join(this._currentWorkingDir, "playwright-report/trace");
+        const traceExists = await (0, util_1.checkFileExistsAsync)(traceFilePath);
+        if (traceExists) {
+            logger_1.logger.info("[Empirical Reporter] Uploading Trace folder");
+            const uploadTraceTask = (0, uploader_1.createUploadTaskV2)({
+                sourceDir: traceFilePath,
+                destinationDir: path_1.default.join(this._destinationDir, "trace"),
+                uploadBucket: this._uploadBucket,
+            });
+            void (0, queue_1.sendTaskToQueue)(uploadTraceTask);
+        }
+        else {
+            logger_1.logger.error(`[Empirical Reporter] playwright-report/trace does not exist at: ${traceFilePath}`);
+            return;
+        }
+        await (0, queue_1.waitForTaskQueueToFinish)();
+        logger_1.logger.info("[Empirical Reporter] All uploads finished");
+        const endTime = new Date().getTime();
+        logger_1.logger.info("[Empirical Reporter] Finished final report upload at: ", new Intl.DateTimeFormat("en-US", {
+            hour: "2-digit",
+            minute: "2-digit",
+            second: "2-digit",
+        }).format(endTime));
+        const timeDiff = endTime - startTime;
+        logger_1.logger.info("[Empirical Reporter] Time taken to upload after tests ended: ", timeDiff, "ms");
+    }
+}
+exports.default = EmpiricalReporter;

package/dist/reporter/types.d.ts

@@ -1,17 +1,3 @@
-import { TestCase, TestResult } from "@playwright/test/reporter";
-export type VideoURL = {
-    name: string;
-    url: string;
-};
-export type TestCaseRunEndEvent = {
-    test: TestCase;
-    result: TestResult;
-    assetsURL: {
-        trace: string;
-        videos: VideoURL[];
-    };
-    suites: string[];
-    projectName: string;
-    runId: string;
-};
+import type { TestCaseRunEndEvent as SharedTestCaseRunEndEvent } from "@empiricalrun/shared-types";
+export type TestCaseRunEndEvent = SharedTestCaseRunEndEvent;
 //# sourceMappingURL=types.d.ts.map

package/dist/reporter/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/reporter/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE;QACT,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,QAAQ,EAAE,CAAC;KACpB,CAAC;IACF,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/reporter/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,IAAI,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEnG,MAAM,MAAM,mBAAmB,GAAG,yBAAyB,CAAC"}

package/dist/reporter/uploader.d.ts

@@ -10,4 +10,24 @@ export declare function createUploadTask({ fileList, outputFolder, relativePath,
     fileList?: string[];
     relativePath?: string;
 }): AsyncTask;
+/**
+ * Creates an async task for uploading files to R2 storage.
+ * This is an improved version of createUploadTask that:
+ * 1. Takes sourceDir directly instead of computing it from outputFolder + relativePath
+ * 2. Allows specifying a custom upload bucket instead of using a global constant
+ * 3. Preserves the full file path in the returned URLs instead of just the filename
+ *
+ * @param {Object} params - The upload parameters
+ * @param {string} params.sourceDir - The source directory containing files to upload
+ * @param {string[]} [params.fileList] - Optional list of specific files to upload
+ * @param {string} params.destinationDir - The destination directory in R2 storage
+ * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
+ */
+export declare function createUploadTaskV2({ sourceDir, fileList, destinationDir, uploadBucket, }: {
+    sourceDir: string;
+    fileList?: string[];
+    destinationDir: string;
+    uploadBucket: string;
+}): AsyncTask;
 //# sourceMappingURL=uploader.d.ts.map

package/dist/reporter/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAmB,MAAM,2BAA2B,CAAC;AAIrE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAKpC,wBAAsB,WAAW,CAAC,EAChC,QAAQ,EACR,YAAY,EACZ,YAAY,GACb,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAwB1B;AAED,wBAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,YAAY,EACZ,YAAiB,GAClB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,SAAS,CAaZ"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAmB,MAAM,2BAA2B,CAAC;AAIrE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAKpC,wBAAsB,WAAW,CAAC,EAChC,QAAQ,EACR,YAAY,EACZ,YAAY,GACb,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAwB1B;AAED,wBAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,YAAY,EACZ,YAAiB,GAClB,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,SAAS,CAaZ;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,EACjC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,GACb,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,SAAS,CA0BZ"}

package/dist/reporter/uploader.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadFiles = uploadFiles;
 exports.createUploadTask = createUploadTask;
+exports.createUploadTaskV2 = createUploadTaskV2;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
 const path_1 = __importDefault(require("path"));
 const logger_1 = require("../logger");
@@ -45,3 +46,42 @@ function createUploadTask({ fileList, outputFolder, relativePath = "", }) {
             .catch((e) => logger_1.logger.error("Error uploading files", e));
     };
 }
+/**
+ * Creates an async task for uploading files to R2 storage.
+ * This is an improved version of createUploadTask that:
+ * 1. Takes sourceDir directly instead of computing it from outputFolder + relativePath
+ * 2. Allows specifying a custom upload bucket instead of using a global constant
+ * 3. Preserves the full file path in the returned URLs instead of just the filename
+ *
+ * @param {Object} params - The upload parameters
+ * @param {string} params.sourceDir - The source directory containing files to upload
+ * @param {string[]} [params.fileList] - Optional list of specific files to upload
+ * @param {string} params.destinationDir - The destination directory in R2 storage
+ * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
+ */
+function createUploadTaskV2({ sourceDir, fileList, destinationDir, uploadBucket, }) {
+    return async () => {
+        try {
+            const uploadedFiles = await (0, r2_uploader_1.uploadDirectory)({
+                sourceDir,
+                fileList,
+                destinationDir,
+                uploadBucket,
+            });
+            const urls = {};
+            Object.entries(uploadedFiles).forEach(([filePath]) => {
+                // Example: If R2_BASE_URL is "https://r2.example.com" and destinationDir is "test-results/123" and filePath is "screenshot.png"
+                // The resulting URL will be: "https://r2.example.com/test-results/123/screenshot.png"
+                const fileUrl = new URL(path_1.default.join(destinationDir, filePath), R2_BASE_URL);
+                urls[filePath] = fileUrl.toString();
+            });
+            logger_1.logger.info("Finished Upload Files");
+            return urls;
+        }
+        catch (error) {
+            logger_1.logger.error("Error uploading files", error);
+            return {};
+        }
+    };
+}

package/dist/reporter/util.d.ts

@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { GenericPlaywrightAttachment } from "@empiricalrun/shared-types";
 import { TestCase } from "@playwright/test/reporter";
 import { TestCaseRunEndEvent } from "./types";
 export declare function getPackageJsonPath(folderPath: string): string;
@@ -35,4 +36,34 @@ export declare function suitesAndProjectForTest(test: TestCase): {
 };
 export declare function sendTestCaseUpdateToDashboard(params: TestCaseRunEndEvent): Promise<void>;
 export declare function safelySerialiseJSON(obj: any, keyFilter?: (key: string) => boolean): any;
+export declare function updateHtmlZipFileAttachmentPaths(jsonFilePath: string, htmlFilePath: string): Promise<void>;
+/**
+ * Replaces media paths in test results with normalized paths from the test attachment map.
+ * This function updates the attachments array in each test result with the corresponding
+ * normalized attachments from the test attachment map.
+ *
+ * @param jsonData - The JSON data containing test results, which can be either in the format
+ *                   of { files: [{ tests: [...] }] } or { tests: [...] }
+ * @param testAttachmentMap - A Map where the key is the test ID and the value is an array of
+ *                           objects containing attachments with updated path for each test result
+ * @returns void
+ */
+export declare function replaceMediaPaths(jsonData: any, testAttachmentMap: Map<string, {
+    attachments: GenericPlaywrightAttachment[];
+}[]>): void;
+/**
+ * Builds a map from test ID to an array of objects containing attachments for each result (retry).
+ * The attachments are normalized to use relative paths starting with 'data/'.
+ * @param summaryJsonFilePath Path to the summary JSON file
+ * @returns Map<string, { attachments: GenericPlaywrightAttachment[] }[]>
+ */
+export declare function buildTestAttachmentMapFromSummary(summaryJsonFilePath: string): Map<string, {
+    attachments: GenericPlaywrightAttachment[];
+}[]>;
+/**
+ * Checks asynchronously if a file exists at the given path.
+ * @param filePath - The path to the file to check
+ * @returns Promise<boolean> - true if the file exists, false otherwise
+ */
+export declare function checkFileExistsAsync(filePath: string): Promise<boolean>;
 //# sourceMappingURL=util.d.ts.map

package/dist/reporter/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/reporter/util.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAS,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAgB5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAI9C,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAmB7D;AAED,wBAAgB,yBAAyB,CACvC,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAAG,SAAS,UAMhC;AAED,wBAAsB,0BAA0B,CAC9C,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5E,OAAO,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC,CAyCD;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,QAAQ;;;EAwBrD;AAED,wBAAsB,6BAA6B,CACjD,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAyDf;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,GACnC,GAAG,CAIL"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/reporter/util.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAKL,QAAQ,EACT,MAAM,2BAA2B,CAAC;AAiBnC,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAI9C,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAmB7D;AAED,wBAAgB,yBAAyB,CACvC,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAAG,SAAS,UAMhC;AAED,wBAAsB,0BAA0B,CAC9C,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5E,OAAO,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC,CAyCD;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,QAAQ;;;EAwBrD;AAED,wBAAsB,6BAA6B,CACjD,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAyDf;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,GAAG,EACR,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,GACnC,GAAG,CAIL;AAED,wBAAsB,gCAAgC,CACpD,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,iBAoFrB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,GAAG,EACb,iBAAiB,EAAE,GAAG,CACpB,MAAM,EACN;IAAE,WAAW,EAAE,2BAA2B,EAAE,CAAA;CAAE,EAAE,CACjD,QA+BF;AAED;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAC/C,mBAAmB,EAAE,MAAM,GAC1B,GAAG,CAAC,MAAM,EAAE;IAAE,WAAW,EAAE,2BAA2B,EAAE,CAAA;CAAE,EAAE,CAAC,CAqD/D;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAO7E"}

package/dist/reporter/util.js

@@ -26,6 +26,11 @@ exports.normalizeAndSaveAttachment = normalizeAndSaveAttachment;
 exports.suitesAndProjectForTest = suitesAndProjectForTest;
 exports.sendTestCaseUpdateToDashboard = sendTestCaseUpdateToDashboard;
 exports.safelySerialiseJSON = safelySerialiseJSON;
+exports.updateHtmlZipFileAttachmentPaths = updateHtmlZipFileAttachmentPaths;
+exports.replaceMediaPaths = replaceMediaPaths;
+exports.buildTestAttachmentMapFromSummary = buildTestAttachmentMapFromSummary;
+exports.checkFileExistsAsync = checkFileExistsAsync;
+const adm_zip_1 = __importDefault(require("adm-zip"));
 const async_retry_1 = __importDefault(require("async-retry"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
@@ -166,3 +171,157 @@ async function sendTestCaseUpdateToDashboard(params) {
 function safelySerialiseJSON(obj, keyFilter) {
     return JSON.parse(JSON.stringify(obj, (key, value) => (keyFilter?.(key) ? undefined : value)));
 }
+async function updateHtmlZipFileAttachmentPaths(jsonFilePath, htmlFilePath) {
+    const testAttachmentMap = buildTestAttachmentMapFromSummary(jsonFilePath);
+    let htmlFile;
+    try {
+        htmlFile = await fs_1.default.promises.readFile(htmlFilePath, "utf8");
+    }
+    catch (err) {
+        console.error(`❌ Failed to read HTML file at ${htmlFilePath}:`, err);
+        return;
+    }
+    const base64 = htmlFile.match(/window\.playwrightReportBase64\s*=\s*"(?:data:application\/zip;base64,)?([^"]+)"/)?.[1];
+    if (!base64) {
+        console.error("❌ Base64 zip data not found in HTML.");
+        return;
+    }
+    const htmlDir = path_1.default.dirname(path_1.default.resolve(htmlFilePath));
+    const tempDir = fs_1.default.mkdtempSync(path_1.default.join(htmlDir, "unzipped-"));
+    const zipPath = path_1.default.join(tempDir, "archive.zip");
+    try {
+        await fs_1.default.promises.writeFile(zipPath, Buffer.from(base64, "base64"));
+        new adm_zip_1.default(zipPath).extractAllTo(tempDir, true);
+        await fs_1.default.promises.unlink(zipPath);
+    }
+    catch (err) {
+        console.error("❌ Failed to unzip base64 content:", err);
+        return;
+    }
+    try {
+        const reportPath = path_1.default.join(tempDir, "report.json");
+        const reportData = JSON.parse(await fs_1.default.promises.readFile(reportPath, "utf8"));
+        replaceMediaPaths(reportData, testAttachmentMap);
+        await fs_1.default.promises.writeFile(reportPath, JSON.stringify(reportData, null, 2), "utf8");
+        const files = await fs_1.default.promises.readdir(tempDir);
+        for (const file of files.filter((f) => f.endsWith(".json") && f !== "report.json")) {
+            try {
+                const filePath = path_1.default.join(tempDir, file);
+                const jsonData = JSON.parse(await fs_1.default.promises.readFile(filePath, "utf8"));
+                replaceMediaPaths(jsonData, testAttachmentMap);
+                await fs_1.default.promises.writeFile(filePath, JSON.stringify(jsonData, null, 2), "utf8");
+            }
+            catch (err) {
+                console.error(`❌ Error processing test file ${file}:`, err);
+            }
+        }
+        const newZip = new adm_zip_1.default();
+        newZip.addLocalFolder(tempDir);
+        const newBase64 = newZip.toBuffer().toString("base64");
+        const updatedHtml = htmlFile.replace(/(window\.playwrightReportBase64\s*=\s*")(?:data:application\/zip;base64,)?[^"]*(")/, `$1data:application/zip;base64,${newBase64}$2`);
+        await fs_1.default.promises.writeFile(htmlFilePath, updatedHtml, "utf8");
+        console.log("✅ HTML file updated with new base64 zip attachment.");
+    }
+    catch (err) {
+        console.error("❌ Error processing report.json or updating files:", err);
+    }
+    finally {
+        // Optional cleanup:
+        // await fs.promises.rm(tempDir, { recursive: true, force: true });
+    }
+}
+/**
+ * Replaces media paths in test results with normalized paths from the test attachment map.
+ * This function updates the attachments array in each test result with the corresponding
+ * normalized attachments from the test attachment map.
+ *
+ * @param jsonData - The JSON data containing test results, which can be either in the format
+ *                   of { files: [{ tests: [...] }] } or { tests: [...] }
+ * @param testAttachmentMap - A Map where the key is the test ID and the value is an array of
+ *                           objects containing attachments with updated path for each test result
+ * @returns void
+ */
+function replaceMediaPaths(jsonData, testAttachmentMap) {
+    const testsArray = jsonData.files?.flatMap((f) => f.tests) || jsonData.tests || [];
+    if (!testsArray.length) {
+        console.warn("⚠️ No test entries found in JSON.");
+        return;
+    }
+    for (const test of testsArray) {
+        const testId = test?.testId;
+        if (!testId) {
+            console.warn("⚠️ Test without testId encountered, skipping.");
+            continue;
+        }
+        const mapped = testAttachmentMap.get(testId);
+        if (!mapped) {
+            console.warn(`⚠️ No mapped attachments found for test: "${testId}"`);
+            continue;
+        }
+        // Update each result's attachments directly by index
+        for (let i = 0; i < (test.results || []).length; i++) {
+            const result = test.results[i];
+            const mappedResult = mapped[i];
+            if (!result || !mappedResult)
+                continue;
+            // Overwrite the attachments array directly
+            result.attachments = mappedResult.attachments;
+        }
+    }
+}
+/**
+ * Builds a map from test ID to an array of objects containing attachments for each result (retry).
+ * The attachments are normalized to use relative paths starting with 'data/'.
+ * @param summaryJsonFilePath Path to the summary JSON file
+ * @returns Map<string, { attachments: GenericPlaywrightAttachment[] }[]>
+ */
+function buildTestAttachmentMapFromSummary(summaryJsonFilePath) {
+    const summaryJson = JSON.parse(fs_1.default.readFileSync(summaryJsonFilePath, "utf8"));
+    const testAttachmentMap = new Map();
+    function normalizeAttachmentPath(attachment) {
+        if (!attachment?.path)
+            return attachment;
+        const relativePath = path_1.default.relative(path_1.default.join(process.cwd(), "test-results"), attachment.path);
+        attachment.path = path_1.default.join("data", relativePath);
+        return attachment;
+    }
+    function processSuite(suite) {
+        for (const spec of suite.specs || []) {
+            const testId = spec.id;
+            for (const test of spec.tests) {
+                if (!testId) {
+                    console.warn("⚠️ Test without testId encountered, skipping.");
+                    continue;
+                }
+                const resultAttachments = (test.results || []).map((result) => ({
+                    attachments: (result.attachments || []).map(normalizeAttachmentPath),
+                }));
+                testAttachmentMap.set(testId, [
+                    ...(testAttachmentMap.get(testId) || []),
+                    ...resultAttachments,
+                ]);
+            }
+        }
+        for (const childSuite of suite.suites || []) {
+            processSuite(childSuite);
+        }
+    }
+    for (const suite of summaryJson.suites || []) {
+        processSuite(suite);
+    }
+    return testAttachmentMap;
+}
+/**
+ * Checks asynchronously if a file exists at the given path.
+ * @param filePath - The path to the file to check
+ * @returns Promise<boolean> - true if the file exists, false otherwise
+ */
+async function checkFileExistsAsync(filePath) {
+    try {
+        await fs_1.default.promises.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.26.21",
+  "version": "0.27.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -29,20 +29,23 @@
     "@types/console-log-level": "^1.4.5",
     "@types/node": "^20.14.9",
     "@types/serve-handler": "^6.1.4",
+    "@types/adm-zip": "^0.5.7",
     "playwright-core": "1.47.1",
-    "serve-handler": "^6.1.6"
+    "serve-handler": "^6.1.6",
+    "@empiricalrun/shared-types": "0.5.1"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
+    "adm-zip": "^0.5.16",
     "async-retry": "^1.3.3",
     "authenticator": "^1.1.5",
     "console-log-level": "^1.4.1",
     "mailosaur": "^8.6.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.18.0",
+    "@empiricalrun/llm": "^0.18.1",
     "@empiricalrun/r2-uploader": "^0.3.9",
-    "@empiricalrun/test-gen": "^0.65.0"
+    "@empiricalrun/test-gen": "^0.66.0"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/config.ts","./src/email.ts","./src/index.ts","./src/logger.ts","./src/playwright-extensions.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/captcha/vision.ts","./src/devices/types.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/reporter/base.ts","./src/reporter/custom.ts","./src/reporter/queue.ts","./src/reporter/reporterV2.ts","./src/reporter/types.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/reporter/third_party/html-reporter-types.ts","./src/test/constants.ts","./src/test/expect.ts","./src/test/index.ts","./src/test/types.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"],"version":"5.8.3"}
+{"root":["./src/config.ts","./src/email.ts","./src/index.ts","./src/logger.ts","./src/playwright-extensions.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/captcha/vision.ts","./src/devices/types.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/reporter/base.ts","./src/reporter/custom.ts","./src/reporter/empirical-reporter.ts","./src/reporter/queue.ts","./src/reporter/reporterV2.ts","./src/reporter/types.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/reporter/third_party/html-reporter-types.ts","./src/test/constants.ts","./src/test/expect.ts","./src/test/index.ts","./src/test/types.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-highlights.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/mouse-pointer.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/highlight/click.ts","./src/test/scripts/pw-locator-patch/highlight/expect.ts","./src/test/scripts/pw-locator-patch/highlight/hover.ts","./src/test/scripts/pw-locator-patch/highlight/inner-text.ts","./src/test/scripts/pw-locator-patch/highlight/input-value.ts","./src/test/scripts/pw-locator-patch/highlight/is-checked.ts","./src/test/scripts/pw-locator-patch/highlight/is-disabled.ts","./src/test/scripts/pw-locator-patch/highlight/is-editable.ts","./src/test/scripts/pw-locator-patch/highlight/text-content.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"],"version":"5.8.3"}