@empiricalrun/playwright-utils 0.38.2 → 0.38.4

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @empiricalrun/playwright-utils
 
+## 0.38.4
+
+### Patch Changes
+
+- 48126ea: feat: cost tracking for overlay dismissals
+- Updated dependencies [48126ea]
+  - @empiricalrun/test-gen@0.79.1
+  - @empiricalrun/llm@0.25.2
+
+## 0.38.3
+
+### Patch Changes
+
+- cc335d4: fix: inline dashboard client type
+
 ## 0.38.2
 
 ### Patch Changes

package/dist/mailosaur-client.d.ts

@@ -1,4 +1,21 @@
-import type { IDashboardAPIClient } from "@empiricalrun/shared-types/api/base";
+export interface IDashboardAPIClient {
+    request<T>(endpoint: string, options: {
+        params?: Record<string, string>;
+        body?: Record<string, unknown>;
+        method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
+    }): Promise<T>;
+    callGitHubProxy<T>(opts: {
+        method: "GET" | "POST" | "PATCH" | "PUT" | "DELETE";
+        url: string;
+        body?: unknown;
+    }): Promise<T>;
+    callMailosaurProxy<T>(opts: {
+        method: "GET" | "POST" | "PATCH" | "PUT" | "DELETE";
+        path: string;
+        body?: unknown;
+    }): Promise<T>;
+    getBaseUrl(): string;
+}
 type Link = {
     text?: string;
     href?: string;

package/dist/mailosaur-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mailosaur-client.d.ts","sourceRoot":"","sources":["../src/mailosaur-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAE/E,KAAK,IAAI,GAAG;IACV,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAkCF,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,CACA;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAA;CAAE,GACrC;IAAE,MAAM,CAAC,EAAE,KAAK,CAAC;IAAC,SAAS,EAAE,mBAAmB,CAAA;CAAE,CACrD,CAAC;AAEF,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,SAAS,CAAC,CAAsB;IACxC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;gBAEZ,IAAI,EAAE,sBAAsB;YAW1B,WAAW;IA0CnB,YAAY,CAChB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC,cAAc,CAAC;YAqBZ,iBAAiB;YA2EjB,cAAc;IA+B5B,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,KAAK;CAGd"}
+{"version":3,"file":"mailosaur-client.d.ts","sourceRoot":"","sources":["../src/mailosaur-client.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,CAAC,EACP,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;QACP,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/B,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;KACrD,GACA,OAAO,CAAC,CAAC,CAAC,CAAC;IACd,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE;QACvB,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,QAAQ,CAAC;QACpD,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACf,kBAAkB,CAAC,CAAC,EAAE,IAAI,EAAE;QAC1B,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,QAAQ,CAAC;QACpD,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACf,UAAU,IAAI,MAAM,CAAC;CACtB;AAED,KAAK,IAAI,GAAG;IACV,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAkCF,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,GAAG,CACA;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAA;CAAE,GACrC;IAAE,MAAM,CAAC,EAAE,KAAK,CAAC;IAAC,SAAS,EAAE,mBAAmB,CAAA;CAAE,CACrD,CAAC;AAEF,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,SAAS,CAAC,CAAsB;IACxC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;gBAEZ,IAAI,EAAE,sBAAsB;YAW1B,WAAW;IA0CnB,YAAY,CAChB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC,cAAc,CAAC;YAqBZ,iBAAiB;YA2EjB,cAAc;IA+B5B,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,KAAK;CAGd"}

package/dist/telemetry.d.ts

@@ -1,8 +1,17 @@
 import { TelemetryEvent } from "@empiricalrun/shared-types/telemetry";
 export declare function trackEvent(event: TelemetryEvent): Promise<void>;
 export declare function trackOverlayDismissed(opts: {
-    overlayDescription: string;
-    cacheHit: boolean;
+    description: string;
+    cache: boolean;
     success: boolean;
+    testName?: string;
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    cost?: {
+        input: number;
+        output: number;
+    };
 }): void;
 //# sourceMappingURL=telemetry.d.ts.map

package/dist/telemetry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB,GAAG,IAAI,CAaP"}
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,wBAAsB,UAAU,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBrE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C,GAAG,IAAI,CAkBP"}

package/dist/telemetry.js

@@ -31,11 +31,16 @@ function trackOverlayDismissed(opts) {
         name: "overlay.dismissed",
         properties: {
             environment,
-            overlay_description: opts.overlayDescription,
-            cache_hit: opts.cacheHit,
+            overlay_description: opts.description,
+            cache_hit: opts.cache,
             success: opts.success,
             project_name: process.env.PROJECT_NAME ?? "",
             test_run_id: process.env.TEST_RUN_GITHUB_ACTION_ID ?? "",
+            test_name: opts.testName ?? "",
+            tokens_input: opts.tokens?.input ?? 0,
+            tokens_output: opts.tokens?.output ?? 0,
+            cost_input: opts.cost?.input ?? 0,
+            cost_output: opts.cost?.output ?? 0,
         },
     });
 }

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAO9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;YA2CF,QAAQ;CAwCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAGxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAK9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;YAkDF,QAAQ;CA6BvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.js

@@ -9,7 +9,6 @@ const cache_1 = require("./cache");
 const prompt_1 = require("./prompt");
 const utils_1 = require("./utils");
 const ERROR_SUBSTRING_INTERCEPTION = "intercepts pointer events";
-const PREFERRED_AGENT = "openai-cua";
 function isErrorSupported(errorMessage) {
     if (!errorMessage) {
         return false;
@@ -36,6 +35,7 @@ class OverlayDismissAgent {
             // Ignoring this error
         }
         const description = (0, utils_1.overlayDescription)(element) ?? "unknown";
+        const testName = testFn.info().title;
         reporter(`Attempting to auto-dismiss overlay: ${description}`);
         try {
             const text = await (0, utils_1.textContent)(this.page, element);
@@ -46,25 +46,30 @@ class OverlayDismissAgent {
             });
             if (success) {
                 (0, telemetry_1.trackOverlayDismissed)({
-                    overlayDescription: description,
-                    cacheHit: true,
+                    description,
+                    cache: true,
                     success: true,
+                    testName,
                 });
                 return;
             }
-            await this.runAgent(element, text, testFn);
+            const usage = await this.runAgent(element, text, testFn);
             (0, telemetry_1.trackOverlayDismissed)({
-                overlayDescription: description,
-                cacheHit: false,
+                description,
+                cache: false,
                 success: true,
+                testName,
+                tokens: usage.tokens,
+                cost: usage.cost,
             });
         }
         catch (agentError) {
             reporter(`Error during overlay dismissal agent execution: ${agentError.toString()}`);
             (0, telemetry_1.trackOverlayDismissed)({
-                overlayDescription: description,
-                cacheHit: false,
+                description,
+                cache: false,
                 success: false,
+                testName,
             });
             throw originalError;
         }
@@ -72,39 +77,24 @@ class OverlayDismissAgent {
     async runAgent(element, text, testFn) {
         const task = await (0, prompt_1.getTask)(this.page, element, text);
         const trace = (0, utils_1.traceThis)(task, testFn);
-        if (PREFERRED_AGENT === "inhouse-master") {
-            const result = await (0, run_1.createTestUsingMasterAgent)({
-                task,
-                page: this.page,
-                options: {
-                    useActionSpecificAnnotations: true,
-                },
-            });
-            this.cachingCandidates.push({
-                dom: element?.interceptor,
-                text,
-                code: result.code,
-            });
-        }
-        else if (PREFERRED_AGENT === "openai-cua") {
-            const result = await (0, run_1.createTestUsingComputerUseAgent)({
-                page: this.page,
-                task,
-                trace,
-                // We don't want to use page.pause approach here because
-                // overlay dismissal can kick in during a browser agent tool call
-                // and we need app to be able to use page.pause for agent actions.
-                // Currently, page.pause does not cleanup properly, causing it to work
-                // only once in a particular test run.
-                // Ran into this here: https://dash.empirical.run/quizizz-tests/sessions/2825
-                prefersElementFromPointCodegen: true,
-            });
-            this.cachingCandidates.push({
-                dom: element?.interceptor,
-                text,
-                code: result.code,
-            });
-        }
+        const result = await (0, run_1.createTestUsingComputerUseAgent)({
+            page: this.page,
+            task,
+            trace,
+            // We don't want to use page.pause approach here because
+            // overlay dismissal can kick in during a browser agent tool call
+            // and we need app to be able to use page.pause for agent actions.
+            // Currently, page.pause does not cleanup properly, causing it to work
+            // only once in a particular test run.
+            // Ran into this here: https://dash.empirical.run/quizizz-tests/sessions/2825
+            prefersElementFromPointCodegen: true,
+        });
+        this.cachingCandidates.push({
+            dom: element?.interceptor,
+            text,
+            code: result.code,
+        });
+        return result.usage;
     }
 }
 exports.OverlayDismissAgent = OverlayDismissAgent;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.38.2",
+  "version": "0.38.4",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -42,9 +42,9 @@
     "console-log-level": "^1.4.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.25.1",
+    "@empiricalrun/llm": "^0.25.2",
     "@empiricalrun/r2-uploader": "^0.9.0",
-    "@empiricalrun/test-gen": "^0.79.0"
+    "@empiricalrun/test-gen": "^0.79.1"
   },
   "scripts": {
     "dev": "tsc --build --watch",