@empiricalrun/playwright-utils 0.19.25 → 0.19.27

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @empiricalrun/playwright-utils
 
+## 0.19.27
+
+### Patch Changes
+
+- 3cb3120: test: two more tests for overlay dismissals
+- 588c5b3: fix: error handling in overlay dismissals
+- c599f74: test: failing test for survicate overlay dismissal
+
+## 0.19.26
+
+### Patch Changes
+
+- Updated dependencies [6f876ea]
+- Updated dependencies [658451e]
+  - @empiricalrun/test-gen@0.42.18
+  - @empiricalrun/llm@0.9.32
+
 ## 0.19.25
 
 ### Patch Changes

package/README.md

@@ -13,3 +13,11 @@ Playwright utils for test code repos of our customers
   - Captcha
 - [Email automation](./docs/email.md)
 - [Authentication](./docs/auth.md)
+
+## Development
+
+To run locator patching tests
+
+```sh
+pnpm run test-browser --headed
+```

package/dist/overlay-tests/click.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=click.spec.d.ts.map

package/dist/overlay-tests/click.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"click.spec.d.ts","sourceRoot":"","sources":["../../src/overlay-tests/click.spec.ts"],"names":[],"mappings":""}

package/dist/overlay-tests/click.spec.js

@@ -0,0 +1,61 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_1 = require("@playwright/test");
+const http_server_1 = __importDefault(require("http-server"));
+const path_1 = __importDefault(require("path"));
+const test_2 = require("../test");
+let server;
+let PORT = 1234;
+test_1.test.beforeAll(async () => {
+    server = http_server_1.default.createServer({
+        root: path_1.default.join(process.cwd(), "test-data"),
+    });
+    // Start the server
+    await new Promise((resolve) => {
+        server.listen(PORT, () => {
+            console.log(`Server running at http://localhost:${PORT}`);
+            resolve(true);
+        });
+    });
+});
+test_1.test.afterAll(() => {
+    if (server) {
+        server.close();
+    }
+});
+test_1.test.beforeEach(async ({ page }) => {
+    // Patch methods
+    (0, test_2.injectLocatorHighlightScripts)(page);
+});
+(0, test_1.test)("should dismiss survicate for click", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/survey.html`);
+    // Assert that Survicate and button loads
+    await (0, test_1.expect)(page.getByText("Start survey")).toBeVisible();
+    await (0, test_1.expect)(page.getByRole("button", { name: "Target" })).toBeVisible();
+    // Do the click, which should pass
+    await page.getByRole("button", { name: "Target" }).click();
+});
+(0, test_1.test)("should dismiss two-step overlay for click", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/tos.html`);
+    // Assert that button and overlay load
+    await (0, test_1.expect)(page.getByRole("button", { name: "Target" })).toBeVisible();
+    await (0, test_1.expect)(page.getByRole("heading", { name: "Terms of Service" })).toBeVisible();
+    // Do the click, which should pass
+    await page.getByRole("button", { name: "Target" }).click();
+    await (0, test_1.expect)(page.getByRole("heading", { name: "Terms of Service" })).not.toBeVisible();
+});
+(0, test_1.test)("should choose and dismiss the correct overlay", async ({ page }) => {
+    await page.goto(`http://localhost:${PORT}/two-overlays.html`);
+    // Assert that button and overlays load
+    await (0, test_1.expect)(page.getByRole("button", { name: "Target" })).toBeVisible();
+    await (0, test_1.expect)(page.getByText("This is a toast message")).toBeVisible();
+    await (0, test_1.expect)(page.getByRole("button", { name: "Close" })).toBeVisible();
+    // Do the click, which should pass
+    await page.getByRole("button", { name: "Target" }).click();
+    // Assert correct overlay was dismissed
+    await (0, test_1.expect)(page.getByText("This is a toast message")).not.toBeVisible();
+    await (0, test_1.expect)(page.getByRole("button", { name: "Close" })).toBeVisible();
+});

package/dist/test/scripts/pw-locator-patch/highlight/click.d.ts

@@ -1,4 +1,8 @@
 import type { Locator } from "@playwright/test";
+export declare function extractInterceptingElement(errorMessage: string): {
+    interceptor: string;
+    parent: string | undefined;
+} | undefined;
 export declare function patchClick(LocatorClass: Function & {
     prototype: Locator;
 }): void;

package/dist/test/scripts/pw-locator-patch/highlight/click.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/highlight/click.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAKhD,wBAAgB,UAAU,CAAC,YAAY,EAAE,QAAQ,GAAG;IAAE,SAAS,EAAE,OAAO,CAAA;CAAE,QAuDzE"}
+{"version":3,"file":"click.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/highlight/click.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AA6BhD,wBAAgB,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAC3D;IACE,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,GACD,SAAS,CAiCZ;AA8CD,wBAAgB,UAAU,CAAC,YAAY,EAAE,QAAQ,GAAG;IAAE,SAAS,EAAE,OAAO,CAAA;CAAE,QAmCzE"}

package/dist/test/scripts/pw-locator-patch/highlight/click.js

@@ -1,10 +1,111 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.patchClick = void 0;
+exports.patchClick = exports.extractInterceptingElement = void 0;
 const planner_1 = require("@empiricalrun/test-gen/agent/master/planner");
 const run_1 = require("@empiricalrun/test-gen/agent/master/run");
 // Static flag to track if click has been patched
 const isClickPatched = new WeakMap();
+const ERROR_SUBSTRING_INTERCEPTION = "intercepts pointer events";
+function removeAnsiCodes(str) {
+    const ansiRegex = new RegExp(String.raw `\u001b\[[0-9;]*m`, "g");
+    return str.replace(ansiRegex, "");
+}
+function findAllMatches(str, regex) {
+    if (!regex.global) {
+        regex = new RegExp(regex.source, regex.flags + "g");
+    }
+    const matches = [];
+    let match;
+    while ((match = regex.exec(str)) !== null) {
+        matches.push({
+            match: match[0],
+            index: match.index,
+            groups: match.groups || {},
+            captures: match.slice(1),
+        });
+    }
+    return matches;
+}
+function extractInterceptingElement(errorMessage) {
+    // This extract element and parent info from interception error message
+    // Note that error message from the last retry is returned.
+    //
+    // [Depends on playwright error formatting]
+    // Upstream ref for error line
+    // https://github.com/microsoft/playwright/blob/bd74fc496461cc3c76b6d5def48580e2f535dac1/packages/playwright-core/src/server/dom.ts#L354
+    //
+    // Upstream ref for element description (with or without subtree)
+    // https://github.com/microsoft/playwright/blob/bd74fc496461cc3c76b6d5def48580e2f535dac1/packages/playwright-core/src/server/injected/injectedScript.ts#L938
+    const htmlElementRegex = /<([a-z][a-z0-9]*)\s+([^>]*)>/gi;
+    const linesWithElements = errorMessage
+        .split("\n")
+        .map(removeAnsiCodes)
+        .filter((line) => line.includes(ERROR_SUBSTRING_INTERCEPTION));
+    const lastSuchLine = linesWithElements.pop();
+    if (!lastSuchLine) {
+        return undefined;
+    }
+    const elements = findAllMatches(lastSuchLine, htmlElementRegex);
+    if (!elements) {
+        return undefined;
+    }
+    else if (elements.length === 1) {
+        return {
+            interceptor: elements[0].match,
+            parent: undefined,
+        };
+    }
+    else if (elements.length > 1) {
+        return {
+            interceptor: elements[0].match,
+            parent: elements[1]?.match,
+        };
+    }
+}
+exports.extractInterceptingElement = extractInterceptingElement;
+function overlayElementInfo(errorMessage) {
+    try {
+        const elements = extractInterceptingElement(errorMessage);
+        if (elements) {
+            const { interceptor, parent } = elements;
+            if (parent) {
+                return `${interceptor} inside ${parent}`;
+            }
+            else {
+                return `${interceptor}`;
+            }
+        }
+    }
+    catch (e) {
+        // Ignoring errors on this for now, since we're only using element
+        // info for logging.
+    }
+}
+async function runAgentOnOverlay(pageRef) {
+    const plannerResp = await (0, planner_1.runtimePlannerWithScreenshot)({
+        task: "Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately. Also note that you just need to dismiss popup and do nothing else.",
+        conversation: [],
+        page: pageRef,
+    });
+    if (plannerResp.isDone) {
+        throw new Error("No active popup found");
+    }
+    await (0, run_1.createTestUsingMasterAgent)({
+        task: "Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately. Also note that you just need to dismiss popup and do nothing else.",
+        page: pageRef,
+        options: {
+            useActionSpecificAnnotations: true,
+        },
+    });
+}
+function annotateForReport(description) {
+    console.log(`[Overlay dismissal]: ${description}`);
+    //@ts-ignore
+    global.testFn?.info().annotations.push({
+        type: "auto-dismiss-overlay",
+        description,
+    });
+}
 function patchClick(LocatorClass) {
     // Check if already patched
     if (isClickPatched.get(LocatorClass)) {
@@ -16,39 +117,23 @@ function patchClick(LocatorClass) {
         try {
             await originalClick.apply(this, [options]);
         }
-        catch (e) {
-            //e is not typed, setting it as any to avoid ts-ignore
-            // If its a test gen or it's not an intercept error, don't trigger the flow
-            if (!e.message?.includes("intercepts pointer events")) {
-                throw e;
-            }
+        catch (originalError) {
             if (process.env.TEST_GEN_TOKEN) {
-                throw e;
+                // Happening during test-gen, we ignore this
+                throw originalError;
+            }
+            if (!originalError.message?.includes(ERROR_SUBSTRING_INTERCEPTION)) {
+                throw originalError;
+            }
+            annotateForReport(`Attempting to auto-dismiss overlay ${overlayElementInfo(originalError.message)}`);
+            try {
+                await runAgentOnOverlay(this._frame._page);
             }
-            const plannerResp = await (0, planner_1.runtimePlannerWithScreenshot)({
-                task: "Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately. Also note that you just need to dismiss popup and do nothing else.",
-                conversation: [],
-                //Need to add the type for frame
-                //@ts-ignore
-                page: this._frame._page,
-            });
-            if (plannerResp.isDone) {
-                throw new Error("No active popup found. Original error: ", e);
+            catch (agentError) {
+                annotateForReport(`Error in overlay dismiss: ${agentError.toString()}`);
+                throw originalError;
             }
-            await (0, run_1.createTestUsingMasterAgent)({
-                task: "Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately. Also note that you just need to dismiss popup and do nothing else.",
-                //Need to add the type for frame
-                //@ts-ignore
-                page: this._frame._page,
-                options: {
-                    useActionSpecificAnnotations: true,
-                },
-            });
-            //@ts-ignore
-            global.testFn?.info().annotations.push({
-                type: "auto-overlay-dismissed",
-                description: "This test run had an overlay that was dismissed during runtime",
-            });
+            annotateForReport("Overlay was dismissed, retrying original action");
             await originalClick.apply(this, [options]);
         }
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.19.25",
+  "version": "0.19.27",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -26,9 +26,11 @@
     "@types/async-retry": "^1.4.8",
     "@types/babel__code-frame": "^7.0.6",
     "@types/console-log-level": "^1.4.5",
+    "@types/http-server": "^0.12.4",
     "@types/md5": "^2.3.5",
     "@types/mime": "3.0.0",
-    "@types/node": "^20.14.9"
+    "@types/node": "^20.14.9",
+    "http-server": "^14.1.1"
   },
   "dependencies": {
     "@babel/code-frame": "^7.24.7",
@@ -40,9 +42,9 @@
     "playwright-core": "1.47.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.9.31",
+    "@empiricalrun/llm": "^0.9.32",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/test-gen": "^0.42.17"
+    "@empiricalrun/test-gen": "^0.42.18"
   },
   "scripts": {
     "dev": "tsc --build --watch",
@@ -50,6 +52,7 @@
     "clean": "tsc --build --clean",
     "lint": "eslint .",
     "test": "vitest run",
+    "test-browser": "npx playwright test",
     "test:watch": "vitest"
   }
 }

package/playwright.config.ts

@@ -0,0 +1,21 @@
+import { defineConfig } from "@playwright/test";
+
+// This config is for browser tests to verify the
+// functionality of features inside playwright-utils
+export default defineConfig({
+  testDir: "./src/overlay-tests",
+  retries: 0,
+  // Test timeout needs to be large enough for master agent etc.
+  // to run overlay dismissal. Action timeout can be small.
+  timeout: 90_000,
+  use: {
+    // Setting user agent for headed browser because Survicate
+    // uses this to detect headless and not render in that scenario
+    userAgent:
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    actionTimeout: 10_000,
+  },
+  expect: {
+    timeout: 10_000,
+  },
+});

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["src/**/*.test.ts"],
+    globals: true,
+  },
+});