@poncho-ai/browser 0.4.0 → 0.5.0

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/browser@0.4.0 build /home/runner/work/poncho-ai/poncho-ai/packages/browser
+> @poncho-ai/browser@0.4.0 build /Users/cesar/Dev/latitude/poncho-ai/packages/browser
 > tsup src/index.ts --format esm --dts
 
 CLI Building entry: src/index.ts
@@ -8,7 +8,7 @@
 CLI Target: es2022
 ESM Build start
 ESM dist/index.js 34.91 KB
-ESM ⚡️ Build success in 62ms
+ESM ⚡️ Build success in 218ms
 DTS Build start
-DTS ⚡️ Build success in 4777ms
+DTS ⚡️ Build success in 5616ms
 DTS dist/index.d.ts 12.44 KB

package/.turbo/turbo-test.log

@@ -0,0 +1,12 @@
+
+> @poncho-ai/browser@0.3.0 test /Users/cesar/Dev/latitude/poncho-ai/packages/browser
+> vitest --passWithNoTests
+
+
+ RUN  v1.6.1 /Users/cesar/Dev/latitude/poncho-ai/packages/browser
+
+include: **/*.{test,spec}.?(c|m)[jt]s?(x)
+exclude:  **/node_modules/**, **/dist/**, **/cypress/**, **/.{idea,git,cache,output,temp}/**, **/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*
+watch exclude:  **/node_modules/**, **/dist/**
+No test files found, exiting with code 0
+

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @poncho-ai/browser
 
+## 0.5.0
+
+### Minor Changes
+
+- [`540c8e6`](https://github.com/cesr/poncho-ai/commit/540c8e6d895a95c2f215deb4af219069543371d9) Thanks [@cesr](https://github.com/cesr)! - Add `browser_click_text` and `browser_execute_js` tools for interacting with elements that don't appear in the accessibility snapshot (e.g. styled divs acting as buttons). Also force new-tab navigations (`window.open`, `target="_blank"`) to stay in the current tab so agents don't lose context.
+
 ## 0.4.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -90,6 +90,12 @@ declare class BrowserSession {
      * Only needs to be called once per browser launch.
      */
     private installContextStealth;
+    /**
+     * Force all new-tab navigations (window.open, target="_blank") to open
+     * in the current tab instead. Agents operate on a single tab at a time
+     * and can't see or interact with popups.
+     */
+    private installSameTabScript;
     /**
      * Override the user-agent via CDP on the current page target.
      * CDP Network.setUserAgentOverride is per-target, so call per-tab.
@@ -122,6 +128,8 @@ declare class BrowserSession {
         title: string;
     }>;
     scroll(conversationId: string, direction: "up" | "down", amount?: number): Promise<void>;
+    clickText(conversationId: string, text: string, exact?: boolean): Promise<void>;
+    executeJs(conversationId: string, script: string): Promise<unknown>;
     closeTab(conversationId: string): Promise<void>;
     navigate(conversationId: string, action: string): Promise<void>;
     startScreencast(conversationId: string, options?: ScreencastOptions): Promise<void>;

package/dist/index.js

@@ -186,6 +186,46 @@ async function getBrowserManagerCtor() {
   return BrowserManagerCtor;
 }
 var MAX_TABS = 8;
+var SAME_TAB_INIT_SCRIPT = `
+(() => {
+  // Override window.open to navigate in-place
+  try {
+    const origOpen = window.open;
+    window.open = function(url, target, features) {
+      if (url) {
+        location.href = url;
+        return window;
+      }
+      return origOpen.call(this, url, target, features);
+    };
+  } catch {}
+
+  // Rewrite target="_blank" on existing and future links
+  try {
+    const rewrite = (el) => {
+      if (el.tagName === 'A' && el.target === '_blank') {
+        el.target = '_self';
+      }
+    };
+    // Catch links already in the DOM
+    document.addEventListener('DOMContentLoaded', () => {
+      document.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+    });
+    // Catch dynamically added links
+    new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        for (const node of m.addedNodes) {
+          if (node.nodeType !== 1) continue;
+          rewrite(node);
+          if (node.querySelectorAll) {
+            node.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+          }
+        }
+      }
+    }).observe(document.documentElement, { childList: true, subtree: true });
+  } catch {}
+})();
+`;
 var BrowserSession = class {
   config;
   sessionId;
@@ -264,6 +304,20 @@ var BrowserSession = class {
       console.warn("[poncho][browser] Failed to install stealth init script:", err?.message ?? err);
     }
   }
+  /**
+   * Force all new-tab navigations (window.open, target="_blank") to open
+   * in the current tab instead. Agents operate on a single tab at a time
+   * and can't see or interact with popups.
+   */
+  async installSameTabScript(mgr) {
+    const ctx = mgr.getContext();
+    if (!ctx) return;
+    try {
+      await ctx.addInitScript({ content: SAME_TAB_INIT_SCRIPT });
+    } catch (err) {
+      console.warn("[poncho][browser] Failed to install same-tab init script:", err?.message ?? err);
+    }
+  }
   /**
    * Override the user-agent via CDP on the current page target.
    * CDP Network.setUserAgentOverride is per-target, so call per-tab.
@@ -310,6 +364,7 @@ var BrowserSession = class {
     if (this.stealthEnabled) {
       await this.installContextStealth(mgr);
     }
+    await this.installSameTabScript(mgr);
     try {
       const cdp = await mgr.getCDPSession();
       await cdp.send("Debugger.disable");
@@ -558,6 +613,30 @@ var BrowserSession = class {
       this.unlock();
     }
   }
+  async clickText(conversationId, text, exact) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const selector = exact ? `text="${text}"` : `text=${text}`;
+      const locator = mgr.getLocator(selector);
+      await locator.click();
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+  async executeJs(conversationId, script) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      return await page.evaluate(script);
+    } finally {
+      this.unlock();
+    }
+  }
   async closeTab(conversationId) {
     await this.lock();
     try {
@@ -907,6 +986,53 @@ function createBrowserTools(getSession, getConversationId) {
         return { clicked: ref };
       }
     },
+    {
+      name: "browser_click_text",
+      description: "Click the first visible element on the page that contains the given text. Use this when an element doesn't appear in the snapshot \u2014 e.g. styled divs acting as buttons. By default matches substring (case-insensitive); set exact=true for exact text match.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          text: {
+            type: "string",
+            description: "The visible text of the element to click"
+          },
+          exact: {
+            type: "boolean",
+            description: "If true, match the exact full text (case-sensitive). Default: false (substring, case-insensitive)."
+          }
+        },
+        required: ["text"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const text = String(input.text ?? "");
+        if (!text) throw new Error("text is required");
+        const exact = input.exact === true;
+        await session.clickText(getConversationId(), text, exact);
+        return { clicked: text, exact };
+      }
+    },
+    {
+      name: "browser_execute_js",
+      description: "Execute JavaScript in the current page context and return the result. Use this to inspect or interact with the DOM when snapshot refs aren't available \u2014 e.g. finding elements by text content, getting bounding boxes, or clicking elements by selector. The script is evaluated via page.evaluate(); return a value to get it back.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          script: {
+            type: "string",
+            description: "JavaScript code to evaluate in the page. Use a return statement or expression to get a result back."
+          }
+        },
+        required: ["script"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const script = String(input.script ?? "");
+        if (!script) throw new Error("script is required");
+        const result = await session.executeJs(getConversationId(), script);
+        return { result: result ?? null };
+      }
+    },
     {
       name: "browser_type",
       description: "Type text into a form field identified by its ref from the last snapshot. This clears the field first, then types the new value.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/browser",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Browser automation for Poncho agents, powered by agent-browser",
   "repository": {
     "type": "git",

package/src/session.ts

@@ -62,6 +62,51 @@ async function getBrowserManagerCtor(): Promise<new () => BrowserManagerInstance
 
 const MAX_TABS = 8;
 
+/**
+ * Init script that forces new-tab navigations (window.open, target="_blank")
+ * to open in the current tab. Runs before page scripts on every navigation.
+ */
+const SAME_TAB_INIT_SCRIPT = `
+(() => {
+  // Override window.open to navigate in-place
+  try {
+    const origOpen = window.open;
+    window.open = function(url, target, features) {
+      if (url) {
+        location.href = url;
+        return window;
+      }
+      return origOpen.call(this, url, target, features);
+    };
+  } catch {}
+
+  // Rewrite target="_blank" on existing and future links
+  try {
+    const rewrite = (el) => {
+      if (el.tagName === 'A' && el.target === '_blank') {
+        el.target = '_self';
+      }
+    };
+    // Catch links already in the DOM
+    document.addEventListener('DOMContentLoaded', () => {
+      document.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+    });
+    // Catch dynamically added links
+    new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        for (const node of m.addedNodes) {
+          if (node.nodeType !== 1) continue;
+          rewrite(node);
+          if (node.querySelectorAll) {
+            node.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+          }
+        }
+      }
+    }).observe(document.documentElement, { childList: true, subtree: true });
+  } catch {}
+})();
+`;
+
 // Per-conversation tab state
 interface ConversationTab {
   tabIndex: number;
@@ -163,6 +208,21 @@ export class BrowserSession {
     }
   }
 
+  /**
+   * Force all new-tab navigations (window.open, target="_blank") to open
+   * in the current tab instead. Agents operate on a single tab at a time
+   * and can't see or interact with popups.
+   */
+  private async installSameTabScript(mgr: BrowserManagerInstance): Promise<void> {
+    const ctx = mgr.getContext();
+    if (!ctx) return;
+    try {
+      await ctx.addInitScript({ content: SAME_TAB_INIT_SCRIPT });
+    } catch (err) {
+      console.warn("[poncho][browser] Failed to install same-tab init script:", (err as Error)?.message ?? err);
+    }
+  }
+
   /**
    * Override the user-agent via CDP on the current page target.
    * CDP Network.setUserAgentOverride is per-target, so call per-tab.
@@ -219,6 +279,9 @@ export class BrowserSession {
       await this.installContextStealth(mgr);
     }
 
+    // Redirect new-tab navigations into the current tab
+    await this.installSameTabScript(mgr);
+
     try {
       const cdp = await mgr.getCDPSession();
       await cdp.send("Debugger.disable");
@@ -475,6 +538,32 @@ export class BrowserSession {
     }
   }
 
+  async clickText(conversationId: string, text: string, exact?: boolean): Promise<void> {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const selector = exact ? `text="${text}"` : `text=${text}`;
+      const locator = mgr.getLocator(selector);
+      await locator.click();
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+
+  async executeJs(conversationId: string, script: string): Promise<unknown> {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      return await page.evaluate(script);
+    } finally {
+      this.unlock();
+    }
+  }
+
   async closeTab(conversationId: string): Promise<void> {
     await this.lock();
     try {

package/src/tools.ts

@@ -73,6 +73,62 @@ export function createBrowserTools(
         return { clicked: ref };
       },
     },
+    {
+      name: "browser_click_text",
+      description:
+        "Click the first visible element on the page that contains the given text. " +
+        "Use this when an element doesn't appear in the snapshot — e.g. styled divs acting as buttons. " +
+        "By default matches substring (case-insensitive); set exact=true for exact text match.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          text: {
+            type: "string",
+            description: "The visible text of the element to click",
+          },
+          exact: {
+            type: "boolean",
+            description:
+              "If true, match the exact full text (case-sensitive). Default: false (substring, case-insensitive).",
+          },
+        },
+        required: ["text"],
+      },
+      handler: async (input: BrowserToolInput) => {
+        const session = getSession();
+        const text = String(input.text ?? "");
+        if (!text) throw new Error("text is required");
+        const exact = input.exact === true;
+        await session.clickText(getConversationId(), text, exact);
+        return { clicked: text, exact };
+      },
+    },
+    {
+      name: "browser_execute_js",
+      description:
+        "Execute JavaScript in the current page context and return the result. " +
+        "Use this to inspect or interact with the DOM when snapshot refs aren't available — " +
+        "e.g. finding elements by text content, getting bounding boxes, or clicking elements by selector. " +
+        "The script is evaluated via page.evaluate(); return a value to get it back.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          script: {
+            type: "string",
+            description:
+              "JavaScript code to evaluate in the page. Use a return statement or expression to get a result back.",
+          },
+        },
+        required: ["script"],
+      },
+      handler: async (input: BrowserToolInput) => {
+        const session = getSession();
+        const script = String(input.script ?? "");
+        if (!script) throw new Error("script is required");
+        const result = await session.executeJs(getConversationId(), script);
+        return { result: result ?? null };
+      },
+    },
     {
       name: "browser_type",
       description: