@empiricalrun/playwright-utils 0.63.0 → 0.63.2

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/typesafe.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dismissWithTypeSafe = dismissWithTypeSafe;
+exports.checkHit = checkHit;
+const node_crypto_1 = require("node:crypto");
+const SYSTEM_ONE_URL = "https://api.typesafe.ai/v1/systemone";
+const MODEL = "jev-preview";
+const MAX_STEPS = 4;
+const MIN_POPUP_PROBABILITY = 0.5;
+const MIN_BOX_CONFIDENCE = 0.3;
+async function dismissWithTypeSafe(page, overlay) {
+    const steps = [];
+    const stop = (stopReason) => ({
+        dismissed: stopReason === "popup_gone" && steps.length > 1,
+        stopReason,
+        steps,
+    });
+    let previousScreenshot;
+    for (let i = 0; i < MAX_STEPS; i++) {
+        const viewport = page.viewportSize();
+        if (!viewport) {
+            throw new Error("Viewport size is required for TypeSafe dismissal");
+        }
+        const screenshot = await page.screenshot({ type: "png" });
+        if (previousScreenshot?.equals(screenshot)) {
+            return stop("no_visual_change");
+        }
+        previousScreenshot = screenshot;
+        const startedAt = Date.now();
+        const response = await askSystemOne(screenshot, overlay.text);
+        const { popup_visible, dismiss_control } = response.answers;
+        const step = {
+            popupProbability: popup_visible.noul,
+            box: dismiss_control.box,
+            boxConfidence: dismiss_control.confidence,
+            clicked: undefined,
+            hitCheck: undefined,
+            latencyMs: Date.now() - startedAt,
+            inputTokens: response.usage.input_tokens,
+            outputTokens: response.usage.output_tokens,
+        };
+        steps.push(step);
+        if (popup_visible.noul < MIN_POPUP_PROBABILITY) {
+            return stop("popup_gone");
+        }
+        if (dismiss_control.confidence < MIN_BOX_CONFIDENCE) {
+            return stop("low_box_confidence");
+        }
+        const { box } = dismiss_control;
+        const point = {
+            x: Math.round(((box.left + box.right) / 2) * viewport.width),
+            y: Math.round(((box.top + box.bottom) / 2) * viewport.height),
+        };
+        step.hitCheck = await checkHit(overlay.target, point);
+        if (step.hitCheck !== "ok") {
+            return stop(step.hitCheck);
+        }
+        step.clicked = point;
+        await page.mouse.click(point.x, point.y);
+        await page.waitForTimeout(300);
+    }
+    return stop("max_steps");
+}
+function checkHit(target, point) {
+    return target.evaluate((targetEl, { x, y }) => {
+        const deepHit = (px, py) => {
+            let el = document.elementFromPoint(px, py);
+            while (el?.shadowRoot) {
+                const inner = el.shadowRoot.elementFromPoint(px, py);
+                if (!inner || inner === el) {
+                    break;
+                }
+                el = inner;
+            }
+            return el;
+        };
+        const parentOf = (el) => el.parentNode instanceof ShadowRoot
+            ? el.parentNode.host
+            : el.parentElement;
+        const isAncestorOrSelf = (ancestor, el) => {
+            for (let n = el; n; n = parentOf(n)) {
+                if (n === ancestor) {
+                    return true;
+                }
+            }
+            return false;
+        };
+        const rect = targetEl.getBoundingClientRect();
+        const blocker = deepHit(rect.left + rect.width / 2, rect.top + rect.height / 2);
+        if (!blocker || isAncestorOrSelf(targetEl, blocker)) {
+            return "no_blocker";
+        }
+        let overlayRoot = blocker;
+        for (let p = parentOf(overlayRoot); p && !isAncestorOrSelf(p, targetEl); p = parentOf(p)) {
+            overlayRoot = p;
+        }
+        const hit = deepHit(x, y);
+        if (!hit || !isAncestorOrSelf(overlayRoot, hit)) {
+            return "outside_overlay";
+        }
+        const interactive = 'button, a[href], input, select, label, summary, iframe, [role=button], [role=link], [role=checkbox], [role=menuitem], [role=tab], [onclick], [tabindex]:not([tabindex="-1"])';
+        for (let n = hit; n; n = parentOf(n)) {
+            if (n.matches(interactive) || getComputedStyle(n).cursor === "pointer") {
+                return "ok";
+            }
+            if (n === overlayRoot) {
+                break;
+            }
+        }
+        return "not_clickable";
+    }, point);
+}
+async function askSystemOne(screenshot, overlayText) {
+    const apiKey = process.env.TYPESAFE_API_KEY;
+    if (!apiKey) {
+        throw new Error("TYPESAFE_API_KEY is not set");
+    }
+    const reference = (0, node_crypto_1.randomUUID)();
+    const popup = overlayText
+        ? `the popup that contains the text "${overlayText.trim().slice(0, 300)}"`
+        : "the popup";
+    const body = {
+        model: MODEL,
+        state: {
+            screenshot: reference,
+            context: "A browser test was blocked because a popup (modal, toast, survey, banner or product tour) covers the element it wants to click.",
+        },
+        assets: {
+            [reference]: {
+                type: "image",
+                format: "png",
+                data_base64: screenshot.toString("base64"),
+            },
+        },
+        questions: {
+            popup_visible: {
+                type: "noul",
+                instructions: `Is ${popup} still visible in \`screenshot\`?`,
+            },
+            dismiss_control: {
+                type: "box",
+                instructions: `the single control in \`screenshot\` to click next to dismiss ${popup}: a close (×) button, or a Close, Skip, No thanks, Got it, Accept, Agree or Continue button. Prefer the option that accepts or continues over Decline or Cancel. If a required checkbox must be ticked first, the checkbox. Never the page's own controls behind the popup.`,
+            },
+        },
+    };
+    const response = await fetch(SYSTEM_ONE_URL, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+            "Content-Type": "application/json",
+            "TypeSafe-Danger-Zone-Images": "true",
+        },
+        body: JSON.stringify(body),
+    });
+    if (!response.ok) {
+        throw new Error(`TypeSafe ${response.status}: ${await response.text()}`);
+    }
+    return (await response.json());
+}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.d.ts

@@ -2,6 +2,9 @@ import type { Page } from "@playwright/test";
 import type { TestFn } from "../../types";
 import type { OverlayElement } from "./types";
 export declare function overlayDescription(element: OverlayElement | undefined): string | undefined;
+export declare const DISMISS_OVERLAY_STEP = "Dismiss overlay";
+export declare const DISMISSAL_AGENT_STEP = "Run overlay dismissal agent";
+export declare function dismissOverlayStepTitle(element: OverlayElement | undefined): string;
 export declare function isProductFruitsOverlay(element: OverlayElement | undefined): boolean | undefined;
 export declare function textContent(pageRef: Page, element: OverlayElement | undefined): Promise<string | undefined>;
 export declare function annotateForReport(testFn: TestFn, description: string): void;

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,sBASrE;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,uBAKzE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,+BAgDpC;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,QAMpE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,uDAiBrD"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,sBASrE;AAED,eAAO,MAAM,oBAAoB,oBAAoB,CAAC;AACtD,eAAO,MAAM,oBAAoB,gCAAgC,CAAC;AAElE,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,UAO1E;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,uBAKzE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,cAAc,GAAG,SAAS,+BAgDpC;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,QAMpE;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,uDAiBrD"}

package/dist/test/scripts/pw-locator-patch/dismiss-overlays/utils.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.DISMISSAL_AGENT_STEP = exports.DISMISS_OVERLAY_STEP = void 0;
 exports.overlayDescription = overlayDescription;
+exports.dismissOverlayStepTitle = dismissOverlayStepTitle;
 exports.isProductFruitsOverlay = isProductFruitsOverlay;
 exports.textContent = textContent;
 exports.annotateForReport = annotateForReport;
@@ -17,6 +19,15 @@ function overlayDescription(element) {
         }
     }
 }
+exports.DISMISS_OVERLAY_STEP = "Dismiss overlay";
+exports.DISMISSAL_AGENT_STEP = "Run overlay dismissal agent";
+function dismissOverlayStepTitle(element) {
+    const root = element?.parent ?? element?.interceptor ?? "";
+    const name = /aria-label="([^"]+)"/.exec(root)?.[1] ??
+        /role="([^"]+)"/.exec(root)?.[1] ??
+        /<([a-z][a-z0-9-]*)/i.exec(root)?.[1];
+    return name ? `${exports.DISMISS_OVERLAY_STEP} "${name}"` : exports.DISMISS_OVERLAY_STEP;
+}
 function isProductFruitsOverlay(element) {
     return (element?.parent?.includes("productfruits--container") ||
         element?.interceptor.includes("productfruits--container"));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.63.0",
+  "version": "0.63.2",
   "engines": {
     "node": ">=22"
   },
@@ -37,7 +37,6 @@
     "@playwright/test": "1.63.0",
     "@types/async-retry": "^1.4.8",
     "@types/authenticator": "^1.1.4",
-    "@types/babel__code-frame": "^7.0.6",
     "@types/console-log-level": "^1.4.5",
     "@types/mime": "3.0.0",
     "@types/node": "^22",
@@ -45,22 +44,20 @@
     "openapi-typescript": "7.6.1",
     "playwright-core": "1.63.0",
     "serve-handler": "^6.1.6",
-    "@empiricalrun/shared-types": "0.21.0"
+    "@empiricalrun/shared-types": "0.21.2"
   },
   "dependencies": {
-    "@babel/code-frame": "^7.24.7",
     "async-retry": "^1.3.3",
     "authenticator": "^1.1.5",
     "console-log-level": "^1.4.1",
     "mime": "3.0.0",
-    "openapi-fetch": "0.14.1",
     "openai": "^7.19.0",
+    "openapi-fetch": "0.14.1",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
-    "rimraf": "^6.0.1",
     "ts-morph": "^23.0.0",
-    "@empiricalrun/cua": "^0.5.2",
-    "@empiricalrun/llm": "^0.28.2",
-    "@empiricalrun/r2-uploader": "^0.11.3",
+    "@empiricalrun/cua": "^0.5.4",
+    "@empiricalrun/llm": "^0.28.3",
+    "@empiricalrun/r2-uploader": "^0.11.4",
     "@empiricalrun/reporter": "^0.30.3"
   },
   "scripts": {

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/parity-utils.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/actions/click.ts","./src/test/scripts/pw-locator-patch/actions/hover.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}
+{"version":"7.0.0-dev.20260623.1","root":["./src/api-worker-client.ts","./src/email.ts","./src/inbox-client.ts","./src/index.ts","./src/kv.ts","./src/logger.ts","./src/mailosaur-client.ts","./src/playwright-extensions.ts","./src/telemetry.ts","./src/webhook.ts","./src/auth/google.ts","./src/auth/index.ts","./src/auth/types.ts","./src/captcha/index.ts","./src/config/index.ts","./src/config/proxy.ts","./src/config/devices/types.ts","./src/generated/api-worker-openapi.ts","./src/overlay-tests/cache.spec.ts","./src/overlay-tests/click.spec.ts","./src/overlay-tests/dismiss-mechanics.spec.ts","./src/overlay-tests/dropdown.spec.ts","./src/overlay-tests/fixtures.ts","./src/overlay-tests/patch.spec.ts","./src/overlay-tests/typesafe-hit-check.spec.ts","./src/overlay-tests/typesafe.spec.ts","./src/overlay-tests/visual.spec.ts","./src/reporter/attachment-cleanup-test-reporter.ts","./src/reporter/attachment-cleanup.ts","./src/reporter/blob-utils.ts","./src/reporter/empirical-reporter.ts","./src/reporter/failing-line.ts","./src/reporter/harness.ts","./src/reporter/ibr-utils.ts","./src/reporter/incremental-blob-reporter.ts","./src/reporter/lifecycle-events.ts","./src/reporter/local-test.ts","./src/reporter/parity-utils.ts","./src/reporter/reporter-state.ts","./src/reporter/test-case-sync-reporter.ts","./src/reporter/test-case-sync.ts","./src/reporter/uploader.ts","./src/reporter/util.ts","./src/test/constants.ts","./src/test/coverage.ts","./src/test/index.ts","./src/test/types.ts","./src/test/video-labels.ts","./src/test/expect/index.ts","./src/test/expect/types.ts","./src/test/expect/visual.ts","./src/test/expect/webhook.ts","./src/test/scripts/agent-capabilities.ts","./src/test/scripts/index.ts","./src/test/scripts/locator-vision.ts","./src/test/scripts/types.ts","./src/test/scripts/pw-locator-patch/actions/click.ts","./src/test/scripts/pw-locator-patch/actions/hover.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/cache.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/index.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/prompt.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/types.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/typesafe.ts","./src/test/scripts/pw-locator-patch/dismiss-overlays/utils.ts","./src/test/scripts/pw-locator-patch/utils/index.ts","./src/test/scripts/pw-locator-patch/vision/query.ts"]}