@empiricalrun/playwright-utils 0.26.0 → 0.26.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @empiricalrun/playwright-utils
 
+## 0.26.1
+
+### Patch Changes
+
+- b5a81a1: feat: add test case name to overlay dismissal traces
+- 78e3aa4: fix: overlay prompt should always have general instructions
+- Updated dependencies [99e4e6e]
+- Updated dependencies [cc4cb5e]
+- Updated dependencies [508565d]
+- Updated dependencies [8da022c]
+  - @empiricalrun/test-gen@0.55.0
+  - @empiricalrun/llm@0.15.0
+
 ## 0.26.0
 
 ### Minor Changes

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts

@@ -1,12 +1,13 @@
 import { Page } from "@playwright/test";
-import { OverlayElement } from "./types";
+import type { TestFn } from "../../types";
+import type { OverlayElement } from "./types";
 export declare function isErrorSupported(errorMessage: string | undefined): boolean;
 export declare class OverlayDismissAgent {
     private readonly page;
     private cachingCandidates;
     constructor(page: Page);
     saveToCache(): Promise<void>;
-    run(originalError: any, reporter: (description: string) => void): Promise<void>;
+    run(originalError: any, reporter: (description: string) => void, testFn: TestFn): Promise<void>;
     private runAgent;
 }
 export declare function extractInterceptingElement(errorMessage: string): OverlayElement | undefined;

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAIxC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAOzC,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GACtC,OAAO,CAAC,IAAI,CAAC;YA4BF,QAAQ;CAgCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAO9C,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,WAKhE;AAED,qBAAa,mBAAmB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,iBAAiB,CAIhB;gBAEoB,IAAI,EAAE,IAAI;IAEjC,WAAW;IAMX,GAAG,CACP,aAAa,EAAE,GAAG,EAClB,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,EACvC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;YA4BF,QAAQ;CAiCvB;AAED,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,MAAM,GACnB,cAAc,GAAG,SAAS,CAiC5B"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/index.js

@@ -25,7 +25,7 @@ class OverlayDismissAgent {
             await (0, cache_1.setCodeToCache)(candidate);
         }
     }
-    async run(originalError, reporter) {
+    async run(originalError, reporter, testFn) {
         let element;
         try {
             element = extractInterceptingElement(originalError.message);
@@ -45,15 +45,16 @@ class OverlayDismissAgent {
             if (success) {
                 return;
             }
-            await this.runAgent(element, text);
+            await this.runAgent(element, text, testFn);
         }
         catch (agentError) {
             reporter(`Error during overlay dismissal agent execution: ${agentError.toString()}`);
             throw originalError;
         }
     }
-    async runAgent(element, text) {
+    async runAgent(element, text, testFn) {
         const task = await (0, prompt_1.getTask)(this.page, element, text);
+        const trace = (0, utils_1.traceThis)(task, testFn);
         if (PREFERRED_AGENT === "inhouse-master") {
             const result = await (0, run_1.createTestUsingMasterAgent)({
                 task,
@@ -69,7 +70,6 @@ class OverlayDismissAgent {
             });
         }
         else if (PREFERRED_AGENT === "openai-cua") {
-            const trace = (0, utils_1.traceThis)(task);
             const result = await (0, run_1.createTestUsingComputerUseAgent)({
                 page: this.page,
                 task,

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,cAAc,GAAG,SAAS,EACnC,IAAI,EAAE,MAAM,GAAG,SAAS,mBAqCzB"}
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,cAAc,GAAG,SAAS,EACnC,IAAI,EAAE,MAAM,GAAG,SAAS,mBAyCzB"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/prompt.js

@@ -3,15 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getTask = void 0;
 const utils_1 = require("./utils");
 async function getTask(page, element, text) {
-    if ((0, utils_1.isProductFruitsOverlay)(element)) {
-        // TODO: Remove this special handling with app knowledge
-        return promptForPFScenario(text);
-    }
-    else if (await isLeapChadsScenario(page)) {
-        // TODO: Remove this special handling with app knowledge
-        return promptForLeapChadsScenario();
-    }
-    const promptAddition = text
+    let promptAddition = text
         ? `
 In this case, the popup can be identified with its text content:
 
@@ -19,8 +11,18 @@ In this case, the popup can be identified with its text content:
 ${text}
 </popup_text_content>
 `
-        : ``;
+        : "";
+    if ((0, utils_1.isProductFruitsOverlay)(element)) {
+        // TODO: Remove this special handling with app knowledge
+        promptAddition = promptForPFScenario(text);
+    }
+    else if (await isLeapChadsScenario(page)) {
+        // TODO: Remove this special handling with app knowledge
+        promptAddition = promptForLeapChadsScenario();
+    }
     let task = `
+# General instructions for your task
+
 Find a way to dismiss the popup. If the popup is non dismissible or there is no popup then return immediately.
 Do all actions required to dismiss the popup - don't ask for user confirmation.
 
@@ -34,6 +36,8 @@ Examples of typical popups:
 4. Forms that capture traffic sources or persona information. Make up content for the input fields, if that's required to dismiss the popup.
 5. Anything else that can be dismissed by taking a few user actions.
 
+# Instructions for this specific scenario
+
 ${promptAddition}`;
     return task;
 }
@@ -45,12 +49,12 @@ function promptForPFScenario(text) {
 <overlapper_element_text_content>
 ${text}
 </overlapper_element_text_content>`
-        : undefined;
+        : "";
     return `
 We are attempting to do a click action on Target element.
 
 This action is failing because our Target element is covered with another element (called Overlapper). 
-${promptAddition || ""}
+${promptAddition}
 
 The only way to work around this is to Click on some other sidebar link element.
 

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.d.ts

@@ -1,9 +1,9 @@
 import type { Page } from "@playwright/test";
 import type { TestFn } from "../../types";
-import { OverlayElement } from "./types";
+import type { OverlayElement } from "./types";
 export declare function overlayDescription(element: OverlayElement | undefined): string | undefined;
 export declare function isProductFruitsOverlay(element: OverlayElement | undefined): boolean | undefined;
 export declare function textContent(pageRef: Page, element: OverlayElement | undefined): Promise<string | undefined>;
 export declare function annotateForReport(testFn: TestFn, description: string): void;
-export declare function traceThis(task: string): import("@empiricalrun/llm").TraceClient | undefined;
+export declare function traceThis(task: string, testFn: TestFn): import("@empiricalrun/llm").TraceClient | undefined;
 //# sourceMappingURL=utils.d.ts.map

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,sBASrE;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,uBAKzE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,+BAiDpC;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,QAMpE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,uDAgBrC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,sBASrE;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,uBAKzE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,+BAiDpC;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,QAMpE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,uDAiBrD"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.js

@@ -76,10 +76,11 @@ function annotateForReport(testFn, description) {
     });
 }
 exports.annotateForReport = annotateForReport;
-function traceThis(task) {
+function traceThis(task, testFn) {
     const metadata = {
         projectName: process.env.PROJECT_NAME ?? "",
         testRunId: process.env.TEST_RUN_GITHUB_ACTION_ID ?? "",
+        testCase: testFn.info().title,
     };
     const trace = (0, llm_1.createLangfuseTrace)({
         name: "overlay_dismiss",

package/dist/test/scripts/pw-locator-patch/highlight/click.js

@@ -39,7 +39,7 @@ function patchClick(LocatorClass, testFn) {
                         // This will cause our overlay dismissal to not run.
                         throw originalError;
                     }
-                    await overlayDismiss.run(originalError, reporter);
+                    await overlayDismiss.run(originalError, reporter, testFn);
                     // Dismissal attempt finished (implicitly successful if no error thrown)
                     reporter(`Overlay dismissal attempted, retrying original action.`);
                 }

package/dist/test/scripts/pw-locator-patch/highlight/hover.js

@@ -34,7 +34,7 @@ function patchHover(LocatorClass, testFn) {
                     if (!(0, dismiss_overlays_1.isErrorSupported)(originalError.message)) {
                         throw originalError;
                     }
-                    await overlayDismiss.run(originalError, reporter);
+                    await overlayDismiss.run(originalError, reporter, testFn);
                     reporter(`Overlay dismissal attempted, retrying original action.`);
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -38,9 +38,9 @@
     "mailosaur": "^8.6.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "rimraf": "^6.0.1",
-    "@empiricalrun/llm": "^0.14.8",
+    "@empiricalrun/llm": "^0.15.0",
     "@empiricalrun/r2-uploader": "^0.3.8",
-    "@empiricalrun/test-gen": "^0.54.1"
+    "@empiricalrun/test-gen": "^0.55.0"
   },
   "scripts": {
     "dev": "tsc --build --watch",