@agentium/browser 2.0.5 → 2.0.7

package/dist/index.cjs

@@ -259,12 +259,21 @@ var BrowserProvider = class {
   // ── Screenshot ───────────────────────────────────────────────────────
   async screenshot() {
     this.ensurePage();
-    return await this.page.screenshot({ type: "png", fullPage: false });
+    return await this.page.screenshot({
+      type: "png",
+      fullPage: false,
+      scale: "css"
+    });
+  }
+  /** Viewport size in CSS pixels (matches screenshot dimensions). */
+  get viewport() {
+    return this._viewport;
   }
   // ── Interaction (with optional humanize) ─────────────────────────────
   async click(x, y) {
     this.ensurePage();
-    const [fx, fy] = this.jitter(x, y);
+    const [cx, cy] = this.clampToViewport(x, y);
+    const [fx, fy] = this.jitter(cx, cy);
     if (this._humanize?.mouseMovement) {
       await this.humanMouseMove(fx, fy);
     }
@@ -278,9 +287,10 @@ var BrowserProvider = class {
     await this.humanPause();
   }
   async clickAndType(x, y, text) {
-    await this.click(x, y);
+    const [cx, cy] = this.clampToViewport(x, y);
+    await this.click(cx, cy);
     await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
-    const [fx, fy] = this.jitter(x, y);
+    const [fx, fy] = this.jitter(cx, cy);
     await this.page.mouse.click(fx, fy, { clickCount: 3 });
     await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
     await this.type(text);
@@ -309,8 +319,10 @@ var BrowserProvider = class {
   // ── DOM Extraction ───────────────────────────────────────────────────
   async extractDOM(opts) {
     this.ensurePage();
-    const max = opts?.maxElements ?? 80;
+    const max = opts?.maxElements ?? 120;
     const elements = await this.page.evaluate((limit) => {
+      const doc = globalThis.document;
+      const win = globalThis.window;
       const selectors = [
         "a[href]",
         "button",
@@ -321,37 +333,62 @@ var BrowserProvider = class {
         "[role='link']",
         "[role='tab']",
         "[role='menuitem']",
+        "[role='checkbox']",
+        "[role='radio']",
+        "[role='switch']",
+        "[role='combobox']",
+        "[role='option']",
         "[onclick]",
-        "[contenteditable='true']"
+        "[contenteditable='true']",
+        "[tabindex]:not([tabindex='-1'])"
       ];
-      const all = globalThis.document.querySelectorAll(selectors.join(","));
-      const lines = [];
-      let count = 0;
-      const vh = globalThis.window.innerHeight;
-      const vw = globalThis.window.innerWidth;
+      const all = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const vh = win.innerHeight;
+      const vw = win.innerWidth;
+      const entries = [];
+      const seen = /* @__PURE__ */ new Set();
       for (const el of all) {
-        if (count >= limit) break;
+        if (seen.has(el)) continue;
         const rect = el.getBoundingClientRect();
-        if (rect.width === 0 || rect.height === 0) continue;
+        if (rect.width < 4 || rect.height < 4) continue;
         if (rect.bottom < 0 || rect.top > vh) continue;
         if (rect.right < 0 || rect.left > vw) continue;
+        const style = win.getComputedStyle(el);
+        if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
+          continue;
+        }
+        const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
+        const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
+        try {
+          const hit = doc.elementFromPoint(cx, cy);
+          if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
+            continue;
+          }
+        } catch {
+        }
+        seen.add(el);
         const tag = el.tagName.toLowerCase();
         const role = el.getAttribute("role") || tag;
         const type = el.getAttribute("type") || "";
-        const text = (el.textContent || "").trim().slice(0, 80);
+        const innerText = el.innerText || el.textContent || "";
+        const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
         const placeholder = el.getAttribute("placeholder") || "";
         const ariaLabel = el.getAttribute("aria-label") || "";
+        const title = el.getAttribute("title") || "";
+        const name = el.getAttribute("name") || "";
         const href = el.getAttribute("href") || "";
         const value = el.value || "";
-        const cx = Math.round(rect.left + rect.width / 2);
-        const cy = Math.round(rect.top + rect.height / 2);
-        let label = ariaLabel || text || placeholder || value;
+        let label = ariaLabel || text || placeholder || title || value || name;
         if (!label && href) label = href.slice(0, 60);
         if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
-        lines.push(`[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`);
-        count++;
+        entries.push({
+          cx,
+          cy,
+          line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
+        });
       }
-      return lines.join("\n");
+      entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
+      return entries.slice(0, limit).map((e) => e.line).join("\n");
     }, max);
     return elements;
   }
@@ -459,6 +496,18 @@ var BrowserProvider = class {
     const j = this._humanize.clickJitter;
     return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
   }
+  /**
+   * Safety net: clamp coordinates returned by the vision model to the actual
+   * viewport. If a model occasionally returns image-space coordinates from a
+   * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
+   * from clicking at e.g. (2200, 1300) and either erroring or landing on a
+   * random off-screen element.
+   */
+  clampToViewport(x, y) {
+    const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
+    const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
+    return [cx, cy];
+  }
   /**
    * Simulate human mouse movement using Bézier-like interpolation.
    * Moves from the current mouse position to the target in small steps.
@@ -502,8 +551,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
   const lines = [
     `You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
     ``,
-    `## Viewport`,
-    `The browser viewport is ${viewport.width}\xD7${viewport.height} pixels. All coordinates you provide must be within this range.`,
+    `## Viewport & Coordinate System`,
+    `- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
+    `- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
+    `- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
+    `- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
+    `- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
     ``,
     `## Available Actions`,
     `Respond with a JSON object containing one of these actions:`,
@@ -544,14 +597,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
     `## Rules`,
     `1. ALWAYS look at the screenshot carefully before deciding your action.`,
     `2. Provide coordinates that target the CENTER of the element you want to interact with.`,
-    `3. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
-    `4. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
-    `5. If your previous action didn't produce the expected result, try a different approach.`,
-    `6. If a page is loading or blank, use "wait" with a short delay and try again.`,
-    `7. If you see a cookie banner or popup, dismiss it first before proceeding with the task.`,
-    `8. NEVER hallucinate content. Only report what you can actually see on the screen.`,
-    `9. When the task is fully complete, use "done" immediately with a comprehensive result.`,
-    `10. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
+    `3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
+    `4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
+    `5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
+    `6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
+    `7. If a page is loading or blank, use "wait" with a short delay and try again.`,
+    `8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
+    `9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
+    `10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
+    `11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
     ``,
     `## Response Format`,
     `Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
@@ -657,7 +711,7 @@ var BrowserAgent = class {
     this.defaultStartUrl = config.startUrl;
     this.waitAfterAction = config.waitAfterAction ?? 1500;
     this.maxRepeats = config.maxRepeats ?? 3;
-    this.useDOM = config.useDOM ?? false;
+    this.useDOM = config.useDOM ?? true;
     this.storageState = config.storageState;
     this.recordVideo = config.recordVideo;
     this.credentials = config.credentials;

package/dist/index.d.cts

@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
     /** Max consecutive identical actions before the agent auto-fails. Default: 3 */
     maxRepeats?: number;
     /**
-     * Include a simplified DOM/accessibility tree alongside the screenshot.
-     * Helps the model target elements more accurately. Default: false
+     * Include a simplified DOM/accessibility tree (each interactive element
+     * tagged with its exact center coordinates) alongside the screenshot.
+     * Dramatically improves click accuracy — strongly recommended.
+     * Default: true
      */
     useDOM?: boolean;
     /**
@@ -290,6 +292,11 @@ declare class BrowserProvider {
     navigate(url: string): Promise<void>;
     back(): Promise<void>;
     screenshot(): Promise<Buffer>;
+    /** Viewport size in CSS pixels (matches screenshot dimensions). */
+    get viewport(): {
+        width: number;
+        height: number;
+    };
     click(x: number, y: number): Promise<void>;
     type(text: string): Promise<void>;
     clickAndType(x: number, y: number, text: string): Promise<void>;
@@ -314,6 +321,14 @@ declare class BrowserProvider {
     close(): Promise<void>;
     /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
     private jitter;
+    /**
+     * Safety net: clamp coordinates returned by the vision model to the actual
+     * viewport. If a model occasionally returns image-space coordinates from a
+     * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
+     * from clicking at e.g. (2200, 1300) and either erroring or landing on a
+     * random off-screen element.
+     */
+    private clampToViewport;
     /**
      * Simulate human mouse movement using Bézier-like interpolation.
      * Moves from the current mouse position to the target in small steps.

package/dist/index.d.ts

@@ -91,8 +91,10 @@ interface BrowserAgentConfig {
     /** Max consecutive identical actions before the agent auto-fails. Default: 3 */
     maxRepeats?: number;
     /**
-     * Include a simplified DOM/accessibility tree alongside the screenshot.
-     * Helps the model target elements more accurately. Default: false
+     * Include a simplified DOM/accessibility tree (each interactive element
+     * tagged with its exact center coordinates) alongside the screenshot.
+     * Dramatically improves click accuracy — strongly recommended.
+     * Default: true
      */
     useDOM?: boolean;
     /**
@@ -290,6 +292,11 @@ declare class BrowserProvider {
     navigate(url: string): Promise<void>;
     back(): Promise<void>;
     screenshot(): Promise<Buffer>;
+    /** Viewport size in CSS pixels (matches screenshot dimensions). */
+    get viewport(): {
+        width: number;
+        height: number;
+    };
     click(x: number, y: number): Promise<void>;
     type(text: string): Promise<void>;
     clickAndType(x: number, y: number, text: string): Promise<void>;
@@ -314,6 +321,14 @@ declare class BrowserProvider {
     close(): Promise<void>;
     /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
     private jitter;
+    /**
+     * Safety net: clamp coordinates returned by the vision model to the actual
+     * viewport. If a model occasionally returns image-space coordinates from a
+     * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
+     * from clicking at e.g. (2200, 1300) and either erroring or landing on a
+     * random off-screen element.
+     */
+    private clampToViewport;
     /**
      * Simulate human mouse movement using Bézier-like interpolation.
      * Moves from the current mouse position to the target in small steps.

package/dist/index.js

@@ -221,12 +221,21 @@ var BrowserProvider = class {
   // ── Screenshot ───────────────────────────────────────────────────────
   async screenshot() {
     this.ensurePage();
-    return await this.page.screenshot({ type: "png", fullPage: false });
+    return await this.page.screenshot({
+      type: "png",
+      fullPage: false,
+      scale: "css"
+    });
+  }
+  /** Viewport size in CSS pixels (matches screenshot dimensions). */
+  get viewport() {
+    return this._viewport;
   }
   // ── Interaction (with optional humanize) ─────────────────────────────
   async click(x, y) {
     this.ensurePage();
-    const [fx, fy] = this.jitter(x, y);
+    const [cx, cy] = this.clampToViewport(x, y);
+    const [fx, fy] = this.jitter(cx, cy);
     if (this._humanize?.mouseMovement) {
       await this.humanMouseMove(fx, fy);
     }
@@ -240,9 +249,10 @@ var BrowserProvider = class {
     await this.humanPause();
   }
   async clickAndType(x, y, text) {
-    await this.click(x, y);
+    const [cx, cy] = this.clampToViewport(x, y);
+    await this.click(cx, cy);
     await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
-    const [fx, fy] = this.jitter(x, y);
+    const [fx, fy] = this.jitter(cx, cy);
     await this.page.mouse.click(fx, fy, { clickCount: 3 });
     await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
     await this.type(text);
@@ -271,8 +281,10 @@ var BrowserProvider = class {
   // ── DOM Extraction ───────────────────────────────────────────────────
   async extractDOM(opts) {
     this.ensurePage();
-    const max = opts?.maxElements ?? 80;
+    const max = opts?.maxElements ?? 120;
     const elements = await this.page.evaluate((limit) => {
+      const doc = globalThis.document;
+      const win = globalThis.window;
       const selectors = [
         "a[href]",
         "button",
@@ -283,37 +295,62 @@ var BrowserProvider = class {
         "[role='link']",
         "[role='tab']",
         "[role='menuitem']",
+        "[role='checkbox']",
+        "[role='radio']",
+        "[role='switch']",
+        "[role='combobox']",
+        "[role='option']",
         "[onclick]",
-        "[contenteditable='true']"
+        "[contenteditable='true']",
+        "[tabindex]:not([tabindex='-1'])"
       ];
-      const all = globalThis.document.querySelectorAll(selectors.join(","));
-      const lines = [];
-      let count = 0;
-      const vh = globalThis.window.innerHeight;
-      const vw = globalThis.window.innerWidth;
+      const all = Array.from(doc.querySelectorAll(selectors.join(",")));
+      const vh = win.innerHeight;
+      const vw = win.innerWidth;
+      const entries = [];
+      const seen = /* @__PURE__ */ new Set();
       for (const el of all) {
-        if (count >= limit) break;
+        if (seen.has(el)) continue;
         const rect = el.getBoundingClientRect();
-        if (rect.width === 0 || rect.height === 0) continue;
+        if (rect.width < 4 || rect.height < 4) continue;
         if (rect.bottom < 0 || rect.top > vh) continue;
         if (rect.right < 0 || rect.left > vw) continue;
+        const style = win.getComputedStyle(el);
+        if (style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none" || parseFloat(style.opacity || "1") < 0.1) {
+          continue;
+        }
+        const cx = Math.round(Math.max(0, Math.min(vw - 1, rect.left + rect.width / 2)));
+        const cy = Math.round(Math.max(0, Math.min(vh - 1, rect.top + rect.height / 2)));
+        try {
+          const hit = doc.elementFromPoint(cx, cy);
+          if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
+            continue;
+          }
+        } catch {
+        }
+        seen.add(el);
         const tag = el.tagName.toLowerCase();
         const role = el.getAttribute("role") || tag;
         const type = el.getAttribute("type") || "";
-        const text = (el.textContent || "").trim().slice(0, 80);
+        const innerText = el.innerText || el.textContent || "";
+        const text = innerText.trim().replace(/\s+/g, " ").slice(0, 80);
         const placeholder = el.getAttribute("placeholder") || "";
         const ariaLabel = el.getAttribute("aria-label") || "";
+        const title = el.getAttribute("title") || "";
+        const name = el.getAttribute("name") || "";
         const href = el.getAttribute("href") || "";
         const value = el.value || "";
-        const cx = Math.round(rect.left + rect.width / 2);
-        const cy = Math.round(rect.top + rect.height / 2);
-        let label = ariaLabel || text || placeholder || value;
+        let label = ariaLabel || text || placeholder || title || value || name;
         if (!label && href) label = href.slice(0, 60);
         if (!label) label = `(${tag}${type ? ` type=${type}` : ""})`;
-        lines.push(`[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`);
-        count++;
+        entries.push({
+          cx,
+          cy,
+          line: `[${cx},${cy}] ${role}${type ? `(${type})` : ""}: "${label}"`
+        });
       }
-      return lines.join("\n");
+      entries.sort((a, b) => a.cy === b.cy ? a.cx - b.cx : a.cy - b.cy);
+      return entries.slice(0, limit).map((e) => e.line).join("\n");
     }, max);
     return elements;
   }
@@ -421,6 +458,18 @@ var BrowserProvider = class {
     const j = this._humanize.clickJitter;
     return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
   }
+  /**
+   * Safety net: clamp coordinates returned by the vision model to the actual
+   * viewport. If a model occasionally returns image-space coordinates from a
+   * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
+   * from clicking at e.g. (2200, 1300) and either erroring or landing on a
+   * random off-screen element.
+   */
+  clampToViewport(x, y) {
+    const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
+    const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
+    return [cx, cy];
+  }
   /**
    * Simulate human mouse movement using Bézier-like interpolation.
    * Moves from the current mouse position to the target in small steps.
@@ -464,8 +513,12 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
   const lines = [
     `You are a browser automation agent. You receive a screenshot of a web browser and decide what action to take next to complete the user's task.`,
     ``,
-    `## Viewport`,
-    `The browser viewport is ${viewport.width}\xD7${viewport.height} pixels. All coordinates you provide must be within this range.`,
+    `## Viewport & Coordinate System`,
+    `- The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels.`,
+    `- The screenshot you are given has EXACTLY the same dimensions: ${viewport.width}\xD7${viewport.height} pixels (top-left = 0,0).`,
+    `- ALL coordinates you return MUST be within 0..${viewport.width - 1} for x and 0..${viewport.height - 1} for y.`,
+    `- Coordinates outside this range will be rejected \u2014 your click will go nowhere.`,
+    `- If a "Interactive elements on page" list is provided below, PREFER its coordinates over your own visual estimation. They are exact element centers in the same coordinate system.`,
     ``,
     `## Available Actions`,
     `Respond with a JSON object containing one of these actions:`,
@@ -506,14 +559,15 @@ function buildSystemPrompt(viewport, extraInstructions, credentialKeys) {
     `## Rules`,
     `1. ALWAYS look at the screenshot carefully before deciding your action.`,
     `2. Provide coordinates that target the CENTER of the element you want to interact with.`,
-    `3. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
-    `4. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
-    `5. If your previous action didn't produce the expected result, try a different approach.`,
-    `6. If a page is loading or blank, use "wait" with a short delay and try again.`,
-    `7. If you see a cookie banner or popup, dismiss it first before proceeding with the task.`,
-    `8. NEVER hallucinate content. Only report what you can actually see on the screen.`,
-    `9. When the task is fully complete, use "done" immediately with a comprehensive result.`,
-    `10. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
+    `3. When an "Interactive elements on page" list is provided, identify the target element in that list by its visible label (button text, aria-label, placeholder, link href) and use ITS coordinates \u2014 they are the ground truth. Only fall back to visual estimation when no matching element is listed.`,
+    `4. For text inputs: click the input field first (using "type" with x/y), then the text will be typed.`,
+    `5. After typing in a search box, you often need to press Enter \u2014 use type with text "\\n" or click the search/submit button.`,
+    `6. If your previous click landed on the wrong element (e.g. page didn't change or a different element became focused), the target was probably misaligned \u2014 re-read the elements list and pick a different coordinate; do NOT repeat the same click.`,
+    `7. If a page is loading or blank, use "wait" with a short delay and try again.`,
+    `8. If you see a cookie banner, consent dialog, or popup, dismiss it FIRST before proceeding with the task \u2014 these often intercept clicks on elements behind them.`,
+    `9. NEVER hallucinate content. Only report what you can actually see on the screen.`,
+    `10. When the task is fully complete, use "done" immediately with a comprehensive result.`,
+    `11. If after several attempts you cannot complete the task, use "fail" with a clear reason.`,
     ``,
     `## Response Format`,
     `Respond with ONLY a valid JSON object. No markdown, no explanation, just the JSON action.`
@@ -619,7 +673,7 @@ var BrowserAgent = class {
     this.defaultStartUrl = config.startUrl;
     this.waitAfterAction = config.waitAfterAction ?? 1500;
     this.maxRepeats = config.maxRepeats ?? 3;
-    this.useDOM = config.useDOM ?? false;
+    this.useDOM = config.useDOM ?? true;
     this.storageState = config.storageState;
     this.recordVideo = config.recordVideo;
     this.credentials = config.credentials;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentium/browser",
-  "version": "2.0.5",
+  "version": "2.0.7",
   "description": "Browser automation agent for Agentium using Playwright",
   "license": "MIT",
   "repository": {
@@ -46,7 +46,7 @@
     "typescript": "^5.6.0"
   },
   "peerDependencies": {
-    "@agentium/core": "^2.0.5",
+    "@agentium/core": "^2.0.7",
     "playwright": ">=1.40.0"
   },
   "peerDependenciesMeta": {