@empiricalrun/playwright-utils 0.38.3 → 0.38.5

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/playwright-utils
 
+## 0.38.5
+
+### Patch Changes
+
+- 86a9e1d: feat: improvements to overlay dismissal telemetry
+
+## 0.38.4
+
+### Patch Changes
+
+- 48126ea: feat: cost tracking for overlay dismissals
+- Updated dependencies [48126ea]
+  - @empiricalrun/test-gen@0.79.1
+  - @empiricalrun/llm@0.25.2
+
 ## 0.38.3
 
 ### Patch Changes

package/dist/telemetry.d.ts

@@ -1,8 +1,19 @@
 import { TelemetryEvent } from "@empiricalrun/shared-types/telemetry";
 export declare function trackEvent(event: TelemetryEvent): Promise<void>;
 export declare function trackOverlayDismissed(opts: {
-    overlayDescription: string;
-    cacheHit: boolean;
+    original: string | undefined;
+    description: string;
+    text: string | undefined;
+    cache: boolean;
     success: boolean;
+    testName?: string;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    cost?: {
+        input: number;
+        output: number;
+    };
 }): void;
 //# sourceMappingURL=telemetry.d.ts.map

package/dist/telemetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB,GAAG,IAAI,CAaP"}
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBrE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,IAAI,CAoBP"}

package/dist/telemetry.js

@@ -4,6 +4,7 @@ exports.trackEvent = trackEvent;
 exports.trackOverlayDismissed = trackOverlayDismissed;
 const TELEMETRY_WORKER_URL = "https://telemetry-worker.empirical-run.workers.dev";
 async function trackEvent(event) {
+    console.log("----", event);
     const apiKey = process.env.EMPIRICAL_TELEMETRY_API_KEY;
     if (!apiKey) {
         return;
@@ -31,11 +32,18 @@ function trackOverlayDismissed(opts) {
         name: "overlay.dismissed",
         properties: {
             environment,
-            overlay_description: opts.overlayDescription,
-            cache_hit: opts.cacheHit,
+            overlay_description: opts.description,
+            overlay_text: opts.text ?? "",
+            original_locator: opts.original ?? "",
+            cache_hit: opts.cache,
             success: opts.success,
             project_name: process.env.PROJECT_NAME ?? "",
             test_run_id: process.env.TEST_RUN_GITHUB_ACTION_ID ?? "",
+            test_name: opts.testName ?? "",
+            tokens_input: opts.tokens?.input ?? 0,
+            tokens_output: opts.tokens?.output ?? 0,
+            cost_input: opts.cost?.input ?? 0,
+            cost_output: opts.cost?.output ?? 0,
         },
     });
 }

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAO9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;YA2CF,QAAQ;CAwCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;YAqDF,QAAQ;CA6BvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.js

@@ -9,7 +9,6 @@ const cache_1 = require("./cache");
 const prompt_1 = require("./prompt");
 const utils_1 = require("./utils");
 const ERROR_SUBSTRING_INTERCEPTION = "intercepts pointer events";
-const PREFERRED_AGENT = "openai-cua";
 function isErrorSupported(errorMessage) {
     if (!errorMessage) {
         return false;
@@ -36,9 +35,16 @@ class OverlayDismissAgent {
             // Ignoring this error
         }
         const description = (0, utils_1.overlayDescription)(element) ?? "unknown";
+        const testName = testFn.info().title;
         reporter(`Attempting to auto-dismiss overlay: ${description}`);
+        const text = await (0, utils_1.textContent)(this.page, element);
+        const eventProps = {
+            description,
+            text,
+            testName,
+            original: originalFailingLocator(originalError.message),
+        };
         try {
-            const text = await (0, utils_1.textContent)(this.page, element);
             const { success } = await (0, cache_1.executeFromCache)({
                 page: this.page,
                 dom: element?.interceptor,
@@ -46,24 +52,26 @@ class OverlayDismissAgent {
             });
             if (success) {
                 (0, telemetry_1.trackOverlayDismissed)({
-                    overlayDescription: description,
-                    cacheHit: true,
+                    ...eventProps,
+                    cache: true,
                     success: true,
                 });
                 return;
             }
-            await this.runAgent(element, text, testFn);
+            const usage = await this.runAgent(element, text, testFn);
             (0, telemetry_1.trackOverlayDismissed)({
-                overlayDescription: description,
-                cacheHit: false,
+                ...eventProps,
+                cache: false,
                 success: true,
+                tokens: usage.tokens,
+                cost: usage.cost,
             });
         }
         catch (agentError) {
             reporter(`Error during overlay dismissal agent execution: ${agentError.toString()}`);
             (0, telemetry_1.trackOverlayDismissed)({
-                overlayDescription: description,
-                cacheHit: false,
+                ...eventProps,
+                cache: false,
                 success: false,
             });
             throw originalError;
@@ -72,39 +80,24 @@ class OverlayDismissAgent {
     async runAgent(element, text, testFn) {
         const task = await (0, prompt_1.getTask)(this.page, element, text);
         const trace = (0, utils_1.traceThis)(task, testFn);
-        if (PREFERRED_AGENT === "inhouse-master") {
-            const result = await (0, run_1.createTestUsingMasterAgent)({
-                task,
-                page: this.page,
-                options: {
-                    useActionSpecificAnnotations: true,
-                },
-            });
-            this.cachingCandidates.push({
-                dom: element?.interceptor,
-                text,
-                code: result.code,
-            });
-        }
-        else if (PREFERRED_AGENT === "openai-cua") {
-            const result = await (0, run_1.createTestUsingComputerUseAgent)({
-                page: this.page,
-                task,
-                trace,
-                // We don't want to use page.pause approach here because
-                // overlay dismissal can kick in during a browser agent tool call
-                // and we need app to be able to use page.pause for agent actions.
-                // Currently, page.pause does not cleanup properly, causing it to work
-                // only once in a particular test run.
-                // Ran into this here: https://dash.empirical.run/quizizz-tests/sessions/2825
-                prefersElementFromPointCodegen: true,
-            });
-            this.cachingCandidates.push({
-                dom: element?.interceptor,
-                text,
-                code: result.code,
-            });
-        }
+        const result = await (0, run_1.createTestUsingComputerUseAgent)({
+            page: this.page,
+            task,
+            trace,
+            // We don't want to use page.pause approach here because
+            // overlay dismissal can kick in during a browser agent tool call
+            // and we need app to be able to use page.pause for agent actions.
+            // Currently, page.pause does not cleanup properly, causing it to work
+            // only once in a particular test run.
+            // Ran into this here: https://dash.empirical.run/quizizz-tests/sessions/2825
+            prefersElementFromPointCodegen: true,
+        });
+        this.cachingCandidates.push({
+            dom: element?.interceptor,
+            text,
+            code: result.code,
+        });
+        return result.usage;
     }
 }
 exports.OverlayDismissAgent = OverlayDismissAgent;
@@ -144,6 +137,15 @@ function extractInterceptingElement(errorMessage) {
         };
     }
 }
+function originalFailingLocator(errorMessage) {
+    try {
+        const p = removeAnsiCodes(errorMessage).split("waiting for ")[1];
+        return p?.split("\n")[0];
+    }
+    catch {
+        return "";
+    }
+}
 function removeAnsiCodes(str) {
     const ansiRegex = new RegExp(String.raw `\u001b\[[0-9;]*m`, "g");
     return str.replace(ansiRegex, "");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.38.3",
+  "version": "0.38.5",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -22,7 +22,7 @@
   },
   "author": "Empirical Team <hey@empirical.run>",
   "devDependencies": {
-    "@playwright/test": "1.53.2",
+    "@playwright/test": "1.57.0",
     "@types/async-retry": "^1.4.8",
     "@types/authenticator": "^1.1.4",
     "@types/babel__code-frame": "^7.0.6",
@@ -30,7 +30,7 @@
     "@types/node": "^20.14.9",
     "@types/serve-handler": "^6.1.4",
     "@types/pg": "^8.11.6",
-    "playwright-core": "1.53.2",
+    "playwright-core": "1.57.0",
     "serve-handler": "^6.1.6",
     "@empiricalrun/shared-types": "0.12.0"
   },
@@ -42,9 +42,9 @@
     "console-log-level": "^1.4.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.25.1",
-    "@empiricalrun/r2-uploader": "^0.9.0",
-    "@empiricalrun/test-gen": "^0.79.0"
+    "@empiricalrun/llm": "^0.25.2",
+    "@empiricalrun/test-gen": "^0.79.1",
+    "@empiricalrun/r2-uploader": "^0.9.0"
   },
   "scripts": {
     "dev": "tsc --build --watch",