@empiricalrun/playwright-utils 0.23.19 → 0.24.0

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/playwright-utils
 
+## 0.24.0
+
+### Minor Changes
+
+- b84661c: feat: use computer-use for overlay dismissals and enable caching
+
+### Patch Changes
+
+- Updated dependencies [ae47eed]
+- Updated dependencies [beb704e]
+- Updated dependencies [3d9c868]
+- Updated dependencies [f283696]
+  - @empiricalrun/test-gen@0.51.5
+  - @empiricalrun/llm@0.11.4
+
 ## 0.23.19
 
 ### Patch Changes

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAYzC,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,iBA+DpC;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B;AAwBD,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,aAAa,EAAE,GAAG,EAClB,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,CAAC,CAAC,CAyBZ"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAYzC,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,iBA8EpC;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B;AAwBD,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,aAAa,EAAE,GAAG,EAClB,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,CAAC,CAAC,CAyBZ"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.js

@@ -5,7 +5,7 @@ const run_1 = require("@empiricalrun/test-gen/agent/master/run");
 const cache_1 = require("./cache");
 const utils_1 = require("./utils");
 const ERROR_SUBSTRING_INTERCEPTION = "intercepts pointer events";
-const PREFERRED_AGENT = "inhouse-master";
+const PREFERRED_AGENT = "openai-cua";
 function isErrorSupported(errorMessage) {
     if (!errorMessage) {
         return false;
@@ -17,7 +17,7 @@ async function runAgentOnOverlay(pageRef, element) {
     const text = await (0, utils_1.textContent)(pageRef, element);
     const promptAddition = text
         ? `
-  The popup can be identified with its text content:
+  In this case, the popup can be identified with its text content:
   
   <popup_text_content>
   ${text}
@@ -27,7 +27,17 @@ async function runAgentOnOverlay(pageRef, element) {
     let task = `
 Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately.
 Do all actions required to dismiss the popup - don't ask for user confirmation.
-Also note that you just need to dismiss popup and do nothing else.
+
+Note that you just need to dismiss popup and DO NOTHING ELSE. If you do more steps, the remainder
+of the workflow will fail, because ONLY the popup is expected to be dismissed by your actions.
+
+Examples of typical popups:
+1. Feature announcements, which can be generally dismissed by close/dismiss buttons.
+2. Product tours which might require going through the tour steps.
+3. Consents for cookies or updated terms of service, which can be dismissed by clicking on the accept/continue button.
+4. Forms that capture traffic sources or persona information. Make up content for the input fields, if that's required to dismiss the popup.
+5. Anything else that can be dismissed by taking a few user actions.
+
 ${promptAddition}`;
     if ((0, utils_1.isProductFruitsOverlay)(element)) {
         // Special handling for product fruits overlay: Overwrite the task
@@ -47,15 +57,16 @@ The only way to work around this is to Click on any other sidebar link element.
 Don't reattempt the click on Target element, your job is done after the first click.
 `;
     }
+    const { success } = await (0, cache_1.executeFromCache)({
+        page: pageRef,
+        dom: element?.interceptor,
+        text,
+    });
+    if (success) {
+        return;
+    }
+    let generatedCode;
     if (PREFERRED_AGENT === "inhouse-master") {
-        const { success } = await (0, cache_1.executeFromCache)({
-            page: pageRef,
-            dom: element?.interceptor,
-            text,
-        });
-        if (success) {
-            return;
-        }
         const result = await (0, run_1.createTestUsingMasterAgent)({
             task,
             page: pageRef,
@@ -63,17 +74,21 @@ Don't reattempt the click on Target element, your job is done after the first cl
                 useActionSpecificAnnotations: true,
             },
         });
-        await (0, cache_1.setCodeToCache)({
-            dom: element?.interceptor,
-            text,
-            code: result.code,
-        });
+        generatedCode = result.code;
     }
     else if (PREFERRED_AGENT === "openai-cua") {
-        await (0, run_1.createTestUsingComputerUseAgent)({
+        const result = await (0, run_1.createTestUsingComputerUseAgent)({
             page: pageRef,
             task,
         });
+        generatedCode = result.code;
+    }
+    if (generatedCode) {
+        await (0, cache_1.setCodeToCache)({
+            dom: element?.interceptor,
+            text,
+            code: generatedCode,
+        });
     }
 }
 exports.runAgentOnOverlay = runAgentOnOverlay;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.23.19",
+  "version": "0.24.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -42,9 +42,9 @@
     "mime": "3.0.0",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.11.3",
+    "@empiricalrun/llm": "^0.11.4",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/test-gen": "^0.51.4"
+    "@empiricalrun/test-gen": "^0.51.5"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/playwright.config.ts

@@ -7,7 +7,7 @@ export default defineConfig({
   retries: 0,
   // Test timeout needs to be large enough for master agent etc.
   // to run overlay dismissal. Action timeout can be small.
-  timeout: 90_000,
+  timeout: 150_000,
   fullyParallel: true,
   workers: "50%",
   use: {