@poncho-ai/browser 0.4.0 → 0.6.0

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/browser@0.4.0 build /home/runner/work/poncho-ai/poncho-ai/packages/browser
+> @poncho-ai/browser@0.6.0 build /Users/cesar/Dev/latitude/poncho-ai/packages/browser
 > tsup src/index.ts --format esm --dts
 
 CLI Building entry: src/index.ts
@@ -7,8 +7,8 @@
 CLI tsup v8.5.1
 CLI Target: es2022
 ESM Build start
-ESM dist/index.js 34.91 KB
-ESM ⚡️ Build success in 62ms
+ESM dist/index.js 40.85 KB
+ESM ⚡️ Build success in 76ms
 DTS Build start
-DTS ⚡️ Build success in 4777ms
-DTS dist/index.d.ts 12.44 KB
+DTS ⚡️ Build success in 1346ms
+DTS dist/index.d.ts 13.55 KB

package/.turbo/turbo-test.log

@@ -0,0 +1,12 @@
+
+> @poncho-ai/browser@0.3.0 test /Users/cesar/Dev/latitude/poncho-ai/packages/browser
+> vitest --passWithNoTests
+
+
+ RUN  v1.6.1 /Users/cesar/Dev/latitude/poncho-ai/packages/browser
+
+include: **/*.{test,spec}.?(c|m)[jt]s?(x)
+exclude:  **/node_modules/**, **/dist/**, **/cypress/**, **/.{idea,git,cache,output,temp}/**, **/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*
+watch exclude:  **/node_modules/**, **/dist/**
+No test files found, exiting with code 0
+

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @poncho-ai/browser
 
+## 0.6.0
+
+### Minor Changes
+
+- [`76294e9`](https://github.com/cesr/poncho-ai/commit/76294e95035bf3abbb19c28871a33f82351c49ec) Thanks [@cesr](https://github.com/cesr)! - Support remote and serverless browser deployments.
+
+  **@poncho-ai/browser**: Add `provider` and `cdpUrl` config options for cloud browser services (Browserbase, Browser Use, Kernel) and direct CDP connections. Auto-detect `@sparticuz/chromium` on serverless platforms (Vercel, Lambda) and default the profile directory to `/tmp`.
+
+  **@poncho-ai/cli**: Generate @vercel/nft trace hints for `@poncho-ai/browser` and `@sparticuz/chromium` in the Vercel entry point so dynamically-loaded browser packages are bundled into the serverless function.
+
+## 0.5.0
+
+### Minor Changes
+
+- [`540c8e6`](https://github.com/cesr/poncho-ai/commit/540c8e6d895a95c2f215deb4af219069543371d9) Thanks [@cesr](https://github.com/cesr)! - Add `browser_click_text` and `browser_execute_js` tools for interacting with elements that don't appear in the accessibility snapshot (e.g. styled divs acting as buttons). Also force new-tab navigations (`window.open`, `target="_blank"`) to stay in the current tab so agents don't lose context.
+
 ## 0.4.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -64,6 +64,13 @@ interface BrowserConfig {
      *  user-agent, and passes anti-automation Chrome flags. */
     stealth?: boolean;
     storagePersistence?: BrowserStoragePersistence;
+    /** Cloud browser provider. Requires the provider's API key env var to be set
+     *  (e.g. `BROWSERBASE_API_KEY` + `BROWSERBASE_PROJECT_ID` for Browserbase).
+     *  When set, the browser runs remotely instead of launching a local Chromium. */
+    provider?: "browserbase" | "browseruse" | "kernel";
+    /** Connect to an existing browser via Chrome DevTools Protocol URL or port.
+     *  Mutually exclusive with `provider`. */
+    cdpUrl?: string;
 }
 
 type FrameListener = (frame: BrowserFrame) => void;
@@ -90,11 +97,24 @@ declare class BrowserSession {
      * Only needs to be called once per browser launch.
      */
     private installContextStealth;
+    /**
+     * Force all new-tab navigations (window.open, target="_blank") to open
+     * in the current tab instead. Agents operate on a single tab at a time
+     * and can't see or interact with popups.
+     */
+    private installSameTabScript;
     /**
      * Override the user-agent via CDP on the current page target.
      * CDP Network.setUserAgentOverride is per-target, so call per-tab.
      */
     private overrideUserAgentOnPage;
+    private get isRemote();
+    private get isServerless();
+    /**
+     * Resolve executablePath for local launches. When no explicit path is set
+     * and we're on a serverless platform, try `@sparticuz/chromium` automatically.
+     */
+    private resolveExecutablePath;
     private launchFreshManager;
     private ensureManager;
     private evictOldestTab;
@@ -122,6 +142,8 @@ declare class BrowserSession {
         title: string;
     }>;
     scroll(conversationId: string, direction: "up" | "down", amount?: number): Promise<void>;
+    clickText(conversationId: string, text: string, exact?: boolean): Promise<void>;
+    executeJs(conversationId: string, script: string): Promise<unknown>;
     closeTab(conversationId: string): Promise<void>;
     navigate(conversationId: string, action: string): Promise<void>;
     startScreencast(conversationId: string, options?: ScreencastOptions): Promise<void>;

package/dist/index.js

@@ -186,6 +186,46 @@ async function getBrowserManagerCtor() {
   return BrowserManagerCtor;
 }
 var MAX_TABS = 8;
+var SAME_TAB_INIT_SCRIPT = `
+(() => {
+  // Override window.open to navigate in-place
+  try {
+    const origOpen = window.open;
+    window.open = function(url, target, features) {
+      if (url) {
+        location.href = url;
+        return window;
+      }
+      return origOpen.call(this, url, target, features);
+    };
+  } catch {}
+
+  // Rewrite target="_blank" on existing and future links
+  try {
+    const rewrite = (el) => {
+      if (el.tagName === 'A' && el.target === '_blank') {
+        el.target = '_self';
+      }
+    };
+    // Catch links already in the DOM
+    document.addEventListener('DOMContentLoaded', () => {
+      document.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+    });
+    // Catch dynamically added links
+    new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        for (const node of m.addedNodes) {
+          if (node.nodeType !== 1) continue;
+          rewrite(node);
+          if (node.querySelectorAll) {
+            node.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+          }
+        }
+      }
+    }).observe(document.documentElement, { childList: true, subtree: true });
+  } catch {}
+})();
+`;
 var BrowserSession = class {
   config;
   sessionId;
@@ -264,6 +304,20 @@ var BrowserSession = class {
       console.warn("[poncho][browser] Failed to install stealth init script:", err?.message ?? err);
     }
   }
+  /**
+   * Force all new-tab navigations (window.open, target="_blank") to open
+   * in the current tab instead. Agents operate on a single tab at a time
+   * and can't see or interact with popups.
+   */
+  async installSameTabScript(mgr) {
+    const ctx = mgr.getContext();
+    if (!ctx) return;
+    try {
+      await ctx.addInitScript({ content: SAME_TAB_INIT_SCRIPT });
+    } catch (err) {
+      console.warn("[poncho][browser] Failed to install same-tab init script:", err?.message ?? err);
+    }
+  }
   /**
    * Override the user-agent via CDP on the current page target.
    * CDP Network.setUserAgentOverride is per-target, so call per-tab.
@@ -284,22 +338,61 @@ var BrowserSession = class {
       console.warn("[poncho][browser] Failed to override UA via CDP:", err?.message ?? err);
     }
   }
+  get isRemote() {
+    return !!(this.config.provider || this.config.cdpUrl);
+  }
+  get isServerless() {
+    return !!(process.env.VERCEL || process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV || process.env.SERVERLESS);
+  }
+  /**
+   * Resolve executablePath for local launches. When no explicit path is set
+   * and we're on a serverless platform, try `@sparticuz/chromium` automatically.
+   */
+  async resolveExecutablePath() {
+    if (this.config.executablePath) return this.config.executablePath;
+    if (!this.isServerless) return void 0;
+    try {
+      const spec = ["@sparticuz", "chromium"].join("/");
+      const mod = await import(
+        /* webpackIgnore: true */
+        spec
+      );
+      const chromium = mod.default ?? mod;
+      const path = await chromium.executablePath();
+      console.log(`[poncho][browser] Auto-detected @sparticuz/chromium: ${path}`);
+      return path;
+    } catch {
+      return void 0;
+    }
+  }
   async launchFreshManager() {
     const Ctor = await getBrowserManagerCtor();
     const mgr = new Ctor();
     const viewport = this.config.viewport ?? { width: 1280, height: 720 };
-    await mkdir(this.profileDir, { recursive: true });
+    const executablePath = await this.resolveExecutablePath();
     const launchOpts = {
       action: "launch",
       headless: this.config.headless ?? true,
       viewport: { width: viewport.width ?? 1280, height: viewport.height ?? 720 },
-      executablePath: this.config.executablePath,
-      profile: this.profileDir
+      executablePath
     };
+    if (this.config.cdpUrl) {
+      launchOpts.cdpUrl = this.config.cdpUrl;
+      console.log(`[poncho][browser] Connecting via CDP: ${this.config.cdpUrl}`);
+    } else if (this.config.provider) {
+      launchOpts.provider = this.config.provider;
+      console.log(`[poncho][browser] Using cloud provider: ${this.config.provider}`);
+    } else {
+      const profileDir = this.isServerless && !this.config.profileDir ? join(tmpdir(), "poncho-browser", this.sessionId) : this.profileDir;
+      await mkdir(profileDir, { recursive: true });
+      launchOpts.profile = profileDir;
+    }
     if (this.stealthEnabled) {
       const ua = this.stealthUserAgent;
       launchOpts.userAgent = ua;
-      launchOpts.args = buildStealthArgs(ua);
+      if (!this.isRemote) {
+        launchOpts.args = buildStealthArgs(ua);
+      }
       console.log("[poncho][browser] Launching with stealth mode enabled (UA: " + ua + ")");
     } else if (this.config.userAgent) {
       launchOpts.userAgent = this.config.userAgent;
@@ -310,6 +403,7 @@ var BrowserSession = class {
     if (this.stealthEnabled) {
       await this.installContextStealth(mgr);
     }
+    await this.installSameTabScript(mgr);
     try {
       const cdp = await mgr.getCDPSession();
       await cdp.send("Debugger.disable");
@@ -558,6 +652,30 @@ var BrowserSession = class {
       this.unlock();
     }
   }
+  async clickText(conversationId, text, exact) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const selector = exact ? `text="${text}"` : `text=${text}`;
+      const locator = mgr.getLocator(selector);
+      await locator.click();
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+  async executeJs(conversationId, script) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      return await page.evaluate(script);
+    } finally {
+      this.unlock();
+    }
+  }
   async closeTab(conversationId) {
     await this.lock();
     try {
@@ -907,6 +1025,53 @@ function createBrowserTools(getSession, getConversationId) {
         return { clicked: ref };
       }
     },
+    {
+      name: "browser_click_text",
+      description: "Click the first visible element on the page that contains the given text. Use this when an element doesn't appear in the snapshot \u2014 e.g. styled divs acting as buttons. By default matches substring (case-insensitive); set exact=true for exact text match.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          text: {
+            type: "string",
+            description: "The visible text of the element to click"
+          },
+          exact: {
+            type: "boolean",
+            description: "If true, match the exact full text (case-sensitive). Default: false (substring, case-insensitive)."
+          }
+        },
+        required: ["text"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const text = String(input.text ?? "");
+        if (!text) throw new Error("text is required");
+        const exact = input.exact === true;
+        await session.clickText(getConversationId(), text, exact);
+        return { clicked: text, exact };
+      }
+    },
+    {
+      name: "browser_execute_js",
+      description: "Execute JavaScript in the current page context and return the result. Use this to inspect or interact with the DOM when snapshot refs aren't available \u2014 e.g. finding elements by text content, getting bounding boxes, or clicking elements by selector. The script is evaluated via page.evaluate(); return a value to get it back.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          script: {
+            type: "string",
+            description: "JavaScript code to evaluate in the page. Use a return statement or expression to get a result back."
+          }
+        },
+        required: ["script"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const script = String(input.script ?? "");
+        if (!script) throw new Error("script is required");
+        const result = await session.executeJs(getConversationId(), script);
+        return { result: result ?? null };
+      }
+    },
     {
       name: "browser_type",
       description: "Type text into a form field identified by its ref from the last snapshot. This clears the field first, then types the new value.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/browser",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Browser automation for Poncho agents, powered by agent-browser",
   "repository": {
     "type": "git",

package/src/session.ts

@@ -62,6 +62,51 @@ async function getBrowserManagerCtor(): Promise<new () => BrowserManagerInstance
 
 const MAX_TABS = 8;
 
+/**
+ * Init script that forces new-tab navigations (window.open, target="_blank")
+ * to open in the current tab. Runs before page scripts on every navigation.
+ */
+const SAME_TAB_INIT_SCRIPT = `
+(() => {
+  // Override window.open to navigate in-place
+  try {
+    const origOpen = window.open;
+    window.open = function(url, target, features) {
+      if (url) {
+        location.href = url;
+        return window;
+      }
+      return origOpen.call(this, url, target, features);
+    };
+  } catch {}
+
+  // Rewrite target="_blank" on existing and future links
+  try {
+    const rewrite = (el) => {
+      if (el.tagName === 'A' && el.target === '_blank') {
+        el.target = '_self';
+      }
+    };
+    // Catch links already in the DOM
+    document.addEventListener('DOMContentLoaded', () => {
+      document.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+    });
+    // Catch dynamically added links
+    new MutationObserver((mutations) => {
+      for (const m of mutations) {
+        for (const node of m.addedNodes) {
+          if (node.nodeType !== 1) continue;
+          rewrite(node);
+          if (node.querySelectorAll) {
+            node.querySelectorAll('a[target="_blank"]').forEach(rewrite);
+          }
+        }
+      }
+    }).observe(document.documentElement, { childList: true, subtree: true });
+  } catch {}
+})();
+`;
+
 // Per-conversation tab state
 interface ConversationTab {
   tabIndex: number;
@@ -163,6 +208,21 @@ export class BrowserSession {
     }
   }
 
+  /**
+   * Force all new-tab navigations (window.open, target="_blank") to open
+   * in the current tab instead. Agents operate on a single tab at a time
+   * and can't see or interact with popups.
+   */
+  private async installSameTabScript(mgr: BrowserManagerInstance): Promise<void> {
+    const ctx = mgr.getContext();
+    if (!ctx) return;
+    try {
+      await ctx.addInitScript({ content: SAME_TAB_INIT_SCRIPT });
+    } catch (err) {
+      console.warn("[poncho][browser] Failed to install same-tab init script:", (err as Error)?.message ?? err);
+    }
+  }
+
   /**
    * Override the user-agent via CDP on the current page target.
    * CDP Network.setUserAgentOverride is per-target, so call per-tab.
@@ -184,25 +244,74 @@ export class BrowserSession {
     }
   }
 
+  private get isRemote(): boolean {
+    return !!(this.config.provider || this.config.cdpUrl);
+  }
+
+  private get isServerless(): boolean {
+    return !!(
+      process.env.VERCEL ||
+      process.env.AWS_LAMBDA_FUNCTION_NAME ||
+      process.env.AWS_EXECUTION_ENV ||
+      process.env.SERVERLESS
+    );
+  }
+
+  /**
+   * Resolve executablePath for local launches. When no explicit path is set
+   * and we're on a serverless platform, try `@sparticuz/chromium` automatically.
+   */
+  private async resolveExecutablePath(): Promise<string | undefined> {
+    if (this.config.executablePath) return this.config.executablePath;
+    if (!this.isServerless) return undefined;
+    try {
+      // Dynamic require — @sparticuz/chromium is an optional peer dependency
+      // that the user installs in their agent project for serverless runtimes.
+      const spec = ["@sparticuz", "chromium"].join("/");
+      const mod = await import(/* webpackIgnore: true */ spec);
+      const chromium = mod.default ?? mod;
+      const path = await chromium.executablePath();
+      console.log(`[poncho][browser] Auto-detected @sparticuz/chromium: ${path}`);
+      return path;
+    } catch {
+      return undefined;
+    }
+  }
+
   private async launchFreshManager(): Promise<BrowserManagerInstance> {
     const Ctor = await getBrowserManagerCtor();
     const mgr = new Ctor();
 
     const viewport = this.config.viewport ?? { width: 1280, height: 720 };
-    await mkdir(this.profileDir, { recursive: true });
+    const executablePath = await this.resolveExecutablePath();
 
     const launchOpts: Record<string, unknown> = {
       action: "launch",
       headless: this.config.headless ?? true,
       viewport: { width: viewport.width ?? 1280, height: viewport.height ?? 720 },
-      executablePath: this.config.executablePath,
-      profile: this.profileDir,
+      executablePath,
     };
 
+    if (this.config.cdpUrl) {
+      launchOpts.cdpUrl = this.config.cdpUrl;
+      console.log(`[poncho][browser] Connecting via CDP: ${this.config.cdpUrl}`);
+    } else if (this.config.provider) {
+      launchOpts.provider = this.config.provider;
+      console.log(`[poncho][browser] Using cloud provider: ${this.config.provider}`);
+    } else {
+      const profileDir = this.isServerless && !this.config.profileDir
+        ? join(tmpdir(), "poncho-browser", this.sessionId)
+        : this.profileDir;
+      await mkdir(profileDir, { recursive: true });
+      launchOpts.profile = profileDir;
+    }
+
     if (this.stealthEnabled) {
       const ua = this.stealthUserAgent!;
       launchOpts.userAgent = ua;
-      launchOpts.args = buildStealthArgs(ua);
+      if (!this.isRemote) {
+        launchOpts.args = buildStealthArgs(ua);
+      }
       console.log("[poncho][browser] Launching with stealth mode enabled (UA: " + ua + ")");
     } else if (this.config.userAgent) {
       launchOpts.userAgent = this.config.userAgent;
@@ -219,6 +328,9 @@ export class BrowserSession {
       await this.installContextStealth(mgr);
     }
 
+    // Redirect new-tab navigations into the current tab
+    await this.installSameTabScript(mgr);
+
     try {
       const cdp = await mgr.getCDPSession();
       await cdp.send("Debugger.disable");
@@ -475,6 +587,32 @@ export class BrowserSession {
     }
   }
 
+  async clickText(conversationId: string, text: string, exact?: boolean): Promise<void> {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const selector = exact ? `text="${text}"` : `text=${text}`;
+      const locator = mgr.getLocator(selector);
+      await locator.click();
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+
+  async executeJs(conversationId: string, script: string): Promise<unknown> {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      return await page.evaluate(script);
+    } finally {
+      this.unlock();
+    }
+  }
+
   async closeTab(conversationId: string): Promise<void> {
     await this.lock();
     try {

package/src/tools.ts

@@ -73,6 +73,62 @@ export function createBrowserTools(
         return { clicked: ref };
       },
     },
+    {
+      name: "browser_click_text",
+      description:
+        "Click the first visible element on the page that contains the given text. " +
+        "Use this when an element doesn't appear in the snapshot — e.g. styled divs acting as buttons. " +
+        "By default matches substring (case-insensitive); set exact=true for exact text match.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          text: {
+            type: "string",
+            description: "The visible text of the element to click",
+          },
+          exact: {
+            type: "boolean",
+            description:
+              "If true, match the exact full text (case-sensitive). Default: false (substring, case-insensitive).",
+          },
+        },
+        required: ["text"],
+      },
+      handler: async (input: BrowserToolInput) => {
+        const session = getSession();
+        const text = String(input.text ?? "");
+        if (!text) throw new Error("text is required");
+        const exact = input.exact === true;
+        await session.clickText(getConversationId(), text, exact);
+        return { clicked: text, exact };
+      },
+    },
+    {
+      name: "browser_execute_js",
+      description:
+        "Execute JavaScript in the current page context and return the result. " +
+        "Use this to inspect or interact with the DOM when snapshot refs aren't available — " +
+        "e.g. finding elements by text content, getting bounding boxes, or clicking elements by selector. " +
+        "The script is evaluated via page.evaluate(); return a value to get it back.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          script: {
+            type: "string",
+            description:
+              "JavaScript code to evaluate in the page. Use a return statement or expression to get a result back.",
+          },
+        },
+        required: ["script"],
+      },
+      handler: async (input: BrowserToolInput) => {
+        const session = getSession();
+        const script = String(input.script ?? "");
+        if (!script) throw new Error("script is required");
+        const result = await session.executeJs(getConversationId(), script);
+        return { result: result ?? null };
+      },
+    },
     {
       name: "browser_type",
       description:

package/src/types.ts

@@ -70,4 +70,11 @@ export interface BrowserConfig {
    *  user-agent, and passes anti-automation Chrome flags. */
   stealth?: boolean;
   storagePersistence?: BrowserStoragePersistence;
+  /** Cloud browser provider. Requires the provider's API key env var to be set
+   *  (e.g. `BROWSERBASE_API_KEY` + `BROWSERBASE_PROJECT_ID` for Browserbase).
+   *  When set, the browser runs remotely instead of launching a local Chromium. */
+  provider?: "browserbase" | "browseruse" | "kernel";
+  /** Connect to an existing browser via Chrome DevTools Protocol URL or port.
+   *  Mutually exclusive with `provider`. */
+  cdpUrl?: string;
 }