@agentium/browser 2.0.7 → 2.0.8

package/dist/index.cjs

@@ -150,7 +150,11 @@ function buildStealthContextOpts(config, viewport) {
     locale: config.locale ?? "en-US",
     timezoneId: config.timezone ?? "America/New_York",
     colorScheme: "light",
-    deviceScaleFactor: 2,
+    // Default to 1 — matches the most common real-user setup and avoids
+    // visual zoom/stretch in headed mode on non-Retina displays (the host
+    // OS would downsample a 2× rendered surface). Users on Retina-only
+    // deployments can opt back into DPR=2 for sharper screenshots.
+    deviceScaleFactor: config.deviceScaleFactor ?? 1,
     hasTouch: false,
     javaScriptEnabled: true,
     ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
@@ -342,7 +346,24 @@ var BrowserProvider = class {
         "[contenteditable='true']",
         "[tabindex]:not([tabindex='-1'])"
       ];
-      const all = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const POINTER_SCAN_CAP = 2e3;
+      const pointerCandidates = [];
+      const allEls = doc.querySelectorAll("*");
+      for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
+        const el = allEls[i];
+        if (!el) continue;
+        const tag = el.tagName.toLowerCase();
+        if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
+        const rect = el.getBoundingClientRect();
+        if (rect.width < 4 || rect.height < 4) continue;
+        if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
+        if (rect.right < 0 || rect.left > win.innerWidth) continue;
+        const style = win.getComputedStyle(el);
+        if (style.cursor !== "pointer") continue;
+        pointerCandidates.push(el);
+      }
+      const all = semantic.concat(pointerCandidates);
       const vh = win.innerHeight;
       const vw = win.innerWidth;
       const entries = [];
@@ -364,6 +385,7 @@ var BrowserProvider = class {
           if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
             continue;
           }
+          if (hit && el.contains(hit)) seen.add(hit);
         } catch {
         }
         seen.add(el);
@@ -392,6 +414,29 @@ var BrowserProvider = class {
     }, max);
     return elements;
   }
+  /**
+   * Deterministic, DOM-based click using Playwright's text locator.
+   *
+   * Returns `true` if a matching, visible, clickable element was found and
+   * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+   * back to coordinate clicking). Substring-matches by default — e.g.
+   * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+   *
+   * This is the same pattern browser-use uses in DOM-only mode and is far
+   * more reliable than coordinate clicks for text-bearing targets.
+   */
+  async clickByText(keyword, opts) {
+    this.ensurePage();
+    const timeout = opts?.timeout ?? 3e3;
+    try {
+      const locator = this.page.locator(`text=${keyword}`).first();
+      await locator.click({ timeout });
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
   // ── Page Info ────────────────────────────────────────────────────────
   async getPageInfo() {
     this.ensurePage();
@@ -564,6 +609,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
     `### click`,
     `Click at a specific coordinate. Use for buttons, links, inputs, checkboxes, etc.`,
     `\`{ "action": "click", "x": <number>, "y": <number>, "description": "<what you are clicking>" }\``,
+    `IMPORTANT: when the target has visible text, ALWAYS put that text in single or double quotes inside the description, e.g. \`"description": "Click on 'Cheapest' tab"\` or \`"description": "Press the \\"Sign in\\" button"\`. The runtime uses the quoted phrase to do a deterministic DOM text-locator click and only falls back to (x,y) if no match is found, which dramatically improves accuracy on dynamic pages.`,
     ``,
     `### type`,
     `Type text. If x/y are provided, click that position first (to focus the input), then type. If omitted, types into the currently focused element. To press Enter after typing (e.g., to submit a search), append "\\n" to the text.`,
@@ -968,9 +1014,20 @@ ${memoryContext}` : memoryContext;
   async executeAction(browser, action) {
     try {
       switch (action.action) {
-        case "click":
-          await browser.click(action.x, action.y);
+        case "click": {
+          const keyword = this.extractClickKeyword(action.description);
+          let clicked = false;
+          if (keyword) {
+            clicked = await browser.clickByText(keyword);
+            if (clicked) {
+              this.logger.debug("Clicked by text", { keyword });
+            }
+          }
+          if (!clicked) {
+            await browser.click(action.x, action.y);
+          }
           break;
+        }
         case "type": {
           const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
           if (action.x != null && action.y != null) {
@@ -1007,6 +1064,52 @@ ${memoryContext}` : memoryContext;
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
+  /**
+   * Parse a quoted target keyword from a click action's `description`.
+   *
+   * The system prompt asks the model to include the visible label of its
+   * click target in quotes (e.g. `"Click on 'Cheapest' tab"`). When a usable
+   * quoted phrase is present we return it so `executeAction` can try a
+   * deterministic Playwright text locator before falling back to
+   * coordinate clicking.
+   *
+   * Returns `undefined` for generic / ambiguous labels (login buttons,
+   * close, OK, etc.) where a substring text match could trivially fire on
+   * the wrong element.
+   */
+  extractClickKeyword(description) {
+    if (!description) return void 0;
+    const match = description.match(/['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/);
+    if (!match) return void 0;
+    const keyword = match[1].trim();
+    if (!keyword || keyword.length < 2) return void 0;
+    const skip = /* @__PURE__ */ new Set([
+      "log in",
+      "login",
+      "sign in",
+      "sign up",
+      "submit",
+      "close",
+      "ok",
+      "okay",
+      "cancel",
+      "yes",
+      "no",
+      "x",
+      "continue",
+      "next",
+      "back",
+      "accept",
+      "dismiss",
+      "got it",
+      "agree",
+      "i agree",
+      "allow",
+      "deny"
+    ]);
+    if (skip.has(keyword.toLowerCase())) return void 0;
+    return keyword;
+  }
 };
 
 // src/credential-vault.ts

package/dist/index.d.cts

@@ -198,6 +198,13 @@ interface StealthConfig {
     };
     /** Ignore HTTPS certificate errors. Default: false (secure). Only enable for local testing. */
     ignoreHTTPSErrors?: boolean;
+    /**
+     * `window.devicePixelRatio` to emulate. Default: `1`. Set to `2` to mimic
+     * a Retina display (sharper screenshots at 2× cost). Setting to 2 on a
+     * non-Retina host display can cause the headed window to look zoomed-out
+     * or stretched because the OS compositor downsamples a 2× surface.
+     */
+    deviceScaleFactor?: number;
     /** HTTP/SOCKS proxy. Format: "http://user:pass@host:port" */
     proxy?: {
         server: string;
@@ -259,6 +266,20 @@ declare class BrowserAgent {
     private finalize;
     private executeAction;
     private sleep;
+    /**
+     * Parse a quoted target keyword from a click action's `description`.
+     *
+     * The system prompt asks the model to include the visible label of its
+     * click target in quotes (e.g. `"Click on 'Cheapest' tab"`). When a usable
+     * quoted phrase is present we return it so `executeAction` can try a
+     * deterministic Playwright text locator before falling back to
+     * coordinate clicking.
+     *
+     * Returns `undefined` for generic / ambiguous labels (login buttons,
+     * close, OK, etc.) where a substring text match could trivially fire on
+     * the wrong element.
+     */
+    private extractClickKeyword;
 }
 
 /**
@@ -305,6 +326,20 @@ declare class BrowserProvider {
     extractDOM(opts?: {
         maxElements?: number;
     }): Promise<string>;
+    /**
+     * Deterministic, DOM-based click using Playwright's text locator.
+     *
+     * Returns `true` if a matching, visible, clickable element was found and
+     * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+     * back to coordinate clicking). Substring-matches by default — e.g.
+     * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+     *
+     * This is the same pattern browser-use uses in DOM-only mode and is far
+     * more reliable than coordinate clicks for text-bearing targets.
+     */
+    clickByText(keyword: string, opts?: {
+        timeout?: number;
+    }): Promise<boolean>;
     getPageInfo(): Promise<PageInfo>;
     waitForStable(minWait?: number): Promise<void>;
     newTab(url?: string): Promise<string>;

package/dist/index.d.ts

@@ -198,6 +198,13 @@ interface StealthConfig {
     };
     /** Ignore HTTPS certificate errors. Default: false (secure). Only enable for local testing. */
     ignoreHTTPSErrors?: boolean;
+    /**
+     * `window.devicePixelRatio` to emulate. Default: `1`. Set to `2` to mimic
+     * a Retina display (sharper screenshots at 2× cost). Setting to 2 on a
+     * non-Retina host display can cause the headed window to look zoomed-out
+     * or stretched because the OS compositor downsamples a 2× surface.
+     */
+    deviceScaleFactor?: number;
     /** HTTP/SOCKS proxy. Format: "http://user:pass@host:port" */
     proxy?: {
         server: string;
@@ -259,6 +266,20 @@ declare class BrowserAgent {
     private finalize;
     private executeAction;
     private sleep;
+    /**
+     * Parse a quoted target keyword from a click action's `description`.
+     *
+     * The system prompt asks the model to include the visible label of its
+     * click target in quotes (e.g. `"Click on 'Cheapest' tab"`). When a usable
+     * quoted phrase is present we return it so `executeAction` can try a
+     * deterministic Playwright text locator before falling back to
+     * coordinate clicking.
+     *
+     * Returns `undefined` for generic / ambiguous labels (login buttons,
+     * close, OK, etc.) where a substring text match could trivially fire on
+     * the wrong element.
+     */
+    private extractClickKeyword;
 }
 
 /**
@@ -305,6 +326,20 @@ declare class BrowserProvider {
     extractDOM(opts?: {
         maxElements?: number;
     }): Promise<string>;
+    /**
+     * Deterministic, DOM-based click using Playwright's text locator.
+     *
+     * Returns `true` if a matching, visible, clickable element was found and
+     * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+     * back to coordinate clicking). Substring-matches by default — e.g.
+     * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+     *
+     * This is the same pattern browser-use uses in DOM-only mode and is far
+     * more reliable than coordinate clicks for text-bearing targets.
+     */
+    clickByText(keyword: string, opts?: {
+        timeout?: number;
+    }): Promise<boolean>;
     getPageInfo(): Promise<PageInfo>;
     waitForStable(minWait?: number): Promise<void>;
     newTab(url?: string): Promise<string>;

package/dist/index.js

@@ -112,7 +112,11 @@ function buildStealthContextOpts(config, viewport) {
     locale: config.locale ?? "en-US",
     timezoneId: config.timezone ?? "America/New_York",
     colorScheme: "light",
-    deviceScaleFactor: 2,
+    // Default to 1 — matches the most common real-user setup and avoids
+    // visual zoom/stretch in headed mode on non-Retina displays (the host
+    // OS would downsample a 2× rendered surface). Users on Retina-only
+    // deployments can opt back into DPR=2 for sharper screenshots.
+    deviceScaleFactor: config.deviceScaleFactor ?? 1,
     hasTouch: false,
     javaScriptEnabled: true,
     ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
@@ -304,7 +308,24 @@ var BrowserProvider = class {
         "[contenteditable='true']",
         "[tabindex]:not([tabindex='-1'])"
       ];
-      const all = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const semantic = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const POINTER_SCAN_CAP = 2e3;
+      const pointerCandidates = [];
+      const allEls = doc.querySelectorAll("*");
+      for (let i = 0; i < allEls.length && pointerCandidates.length < POINTER_SCAN_CAP; i++) {
+        const el = allEls[i];
+        if (!el) continue;
+        const tag = el.tagName.toLowerCase();
+        if (tag === "html" || tag === "body" || tag === "head" || tag === "script" || tag === "style") continue;
+        const rect = el.getBoundingClientRect();
+        if (rect.width < 4 || rect.height < 4) continue;
+        if (rect.bottom < 0 || rect.top > win.innerHeight) continue;
+        if (rect.right < 0 || rect.left > win.innerWidth) continue;
+        const style = win.getComputedStyle(el);
+        if (style.cursor !== "pointer") continue;
+        pointerCandidates.push(el);
+      }
+      const all = semantic.concat(pointerCandidates);
       const vh = win.innerHeight;
       const vw = win.innerWidth;
       const entries = [];
@@ -326,6 +347,7 @@ var BrowserProvider = class {
           if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
             continue;
           }
+          if (hit && el.contains(hit)) seen.add(hit);
         } catch {
         }
         seen.add(el);
@@ -354,6 +376,29 @@ var BrowserProvider = class {
     }, max);
     return elements;
   }
+  /**
+   * Deterministic, DOM-based click using Playwright's text locator.
+   *
+   * Returns `true` if a matching, visible, clickable element was found and
+   * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+   * back to coordinate clicking). Substring-matches by default — e.g.
+   * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+   *
+   * This is the same pattern browser-use uses in DOM-only mode and is far
+   * more reliable than coordinate clicks for text-bearing targets.
+   */
+  async clickByText(keyword, opts) {
+    this.ensurePage();
+    const timeout = opts?.timeout ?? 3e3;
+    try {
+      const locator = this.page.locator(`text=${keyword}`).first();
+      await locator.click({ timeout });
+      await this.humanPause();
+      return true;
+    } catch {
+      return false;
+    }
+  }
   // ── Page Info ────────────────────────────────────────────────────────
   async getPageInfo() {
     this.ensurePage();
@@ -526,6 +571,7 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
     `### click`,
     `Click at a specific coordinate. Use for buttons, links, inputs, checkboxes, etc.`,
     `\`{ "action": "click", "x": <number>, "y": <number>, "description": "<what you are clicking>" }\``,
+    `IMPORTANT: when the target has visible text, ALWAYS put that text in single or double quotes inside the description, e.g. \`"description": "Click on 'Cheapest' tab"\` or \`"description": "Press the \\"Sign in\\" button"\`. The runtime uses the quoted phrase to do a deterministic DOM text-locator click and only falls back to (x,y) if no match is found, which dramatically improves accuracy on dynamic pages.`,
     ``,
     `### type`,
     `Type text. If x/y are provided, click that position first (to focus the input), then type. If omitted, types into the currently focused element. To press Enter after typing (e.g., to submit a search), append "\\n" to the text.`,
@@ -930,9 +976,20 @@ ${memoryContext}` : memoryContext;
   async executeAction(browser, action) {
     try {
       switch (action.action) {
-        case "click":
-          await browser.click(action.x, action.y);
+        case "click": {
+          const keyword = this.extractClickKeyword(action.description);
+          let clicked = false;
+          if (keyword) {
+            clicked = await browser.clickByText(keyword);
+            if (clicked) {
+              this.logger.debug("Clicked by text", { keyword });
+            }
+          }
+          if (!clicked) {
+            await browser.click(action.x, action.y);
+          }
           break;
+        }
         case "type": {
           const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
           if (action.x != null && action.y != null) {
@@ -969,6 +1026,52 @@ ${memoryContext}` : memoryContext;
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
+  /**
+   * Parse a quoted target keyword from a click action's `description`.
+   *
+   * The system prompt asks the model to include the visible label of its
+   * click target in quotes (e.g. `"Click on 'Cheapest' tab"`). When a usable
+   * quoted phrase is present we return it so `executeAction` can try a
+   * deterministic Playwright text locator before falling back to
+   * coordinate clicking.
+   *
+   * Returns `undefined` for generic / ambiguous labels (login buttons,
+   * close, OK, etc.) where a substring text match could trivially fire on
+   * the wrong element.
+   */
+  extractClickKeyword(description) {
+    if (!description) return void 0;
+    const match = description.match(/['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/);
+    if (!match) return void 0;
+    const keyword = match[1].trim();
+    if (!keyword || keyword.length < 2) return void 0;
+    const skip = /* @__PURE__ */ new Set([
+      "log in",
+      "login",
+      "sign in",
+      "sign up",
+      "submit",
+      "close",
+      "ok",
+      "okay",
+      "cancel",
+      "yes",
+      "no",
+      "x",
+      "continue",
+      "next",
+      "back",
+      "accept",
+      "dismiss",
+      "got it",
+      "agree",
+      "i agree",
+      "allow",
+      "deny"
+    ]);
+    if (skip.has(keyword.toLowerCase())) return void 0;
+    return keyword;
+  }
 };
 
 // src/credential-vault.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentium/browser",
-  "version": "2.0.7",
+  "version": "2.0.8",
   "description": "Browser automation agent for Agentium using Playwright",
   "license": "MIT",
   "repository": {
@@ -46,7 +46,7 @@
     "typescript": "^5.6.0"
   },
   "peerDependencies": {
-    "@agentium/core": "^2.0.7",
+    "@agentium/core": "^2.0.8",
     "playwright": ">=1.40.0"
   },
   "peerDependenciesMeta": {