@poncho-ai/browser 0.1.0

package/dist/index.js

@@ -0,0 +1,687 @@
+// src/session.ts
+import { resolve } from "path";
+import { homedir } from "os";
+import { mkdir } from "fs/promises";
+var BrowserManagerCtor;
+async function getBrowserManagerCtor() {
+  if (!BrowserManagerCtor) {
+    const mod = await import("agent-browser/dist/browser.js");
+    BrowserManagerCtor = mod.BrowserManager;
+  }
+  return BrowserManagerCtor;
+}
+var MAX_TABS = 8;
+var BrowserSession = class {
+  config;
+  sessionId;
+  manager;
+  // Tab management: conversationId → tab state
+  tabs = /* @__PURE__ */ new Map();
+  // Serialization lock for tab-switching operations
+  _lockQueue = [];
+  _locked = false;
+  // Currently screencast conversation (only one at a time due to CDP)
+  _screencastConversation;
+  constructor(sessionId, config = {}) {
+    this.sessionId = sessionId;
+    this.config = config;
+  }
+  get profileDir() {
+    return this.config.profileDir ?? resolve(homedir(), ".poncho", "browser-profiles", this.sessionId);
+  }
+  // -----------------------------------------------------------------------
+  // Lock for serializing tab-switching operations
+  // -----------------------------------------------------------------------
+  async lock() {
+    if (!this._locked) {
+      this._locked = true;
+      return;
+    }
+    return new Promise((resolve2, reject) => {
+      const timer = setTimeout(() => {
+        const idx = this._lockQueue.indexOf(resolve2);
+        if (idx !== -1) this._lockQueue.splice(idx, 1);
+        reject(new Error("Browser operation timed out waiting for lock (30s)"));
+      }, 3e4);
+      this._lockQueue.push(() => {
+        clearTimeout(timer);
+        resolve2();
+      });
+    });
+  }
+  unlock() {
+    const next = this._lockQueue.shift();
+    if (next) next();
+    else this._locked = false;
+  }
+  // -----------------------------------------------------------------------
+  // Core browser + tab management
+  // -----------------------------------------------------------------------
+  async launchFreshManager() {
+    const Ctor = await getBrowserManagerCtor();
+    const mgr = new Ctor();
+    const viewport = this.config.viewport ?? { width: 1280, height: 720 };
+    await mkdir(this.profileDir, { recursive: true });
+    await mgr.launch({
+      action: "launch",
+      headless: this.config.headless ?? true,
+      viewport: { width: viewport.width ?? 1280, height: viewport.height ?? 720 },
+      executablePath: this.config.executablePath,
+      profile: this.profileDir
+    });
+    try {
+      const cdp = await mgr.getCDPSession();
+      await cdp.send("Debugger.disable");
+    } catch {
+    }
+    this.manager = mgr;
+    return mgr;
+  }
+  async ensureManager() {
+    if (this.manager) {
+      try {
+        if (this.manager.isLaunched()) return this.manager;
+      } catch {
+      }
+      try {
+        await this.manager.close();
+      } catch {
+      }
+      this.manager = void 0;
+      for (const [cid, tab] of this.tabs) {
+        if (tab.tabIndex >= 0) {
+          tab.tabIndex = -1;
+          tab.active = false;
+          tab.url = void 0;
+        }
+      }
+    }
+    return this.launchFreshManager();
+  }
+  async evictOldestTab(mgr) {
+    let oldest;
+    for (const [cid, tab] of this.tabs) {
+      if (tab.tabIndex < 0) continue;
+      if (!oldest || tab.lastUsed < oldest.tab.lastUsed) {
+        oldest = { cid, tab };
+      }
+    }
+    if (!oldest) return;
+    console.log(`[poncho][browser] Evicting idle tab for conversation ${oldest.cid.slice(0, 8)}...`);
+    if (this._screencastConversation === oldest.cid) {
+      try {
+        await mgr.stopScreencast();
+      } catch {
+      }
+      this._screencastConversation = void 0;
+    }
+    if (this.tabs.size > 1) {
+      try {
+        await mgr.closeTab(oldest.tab.tabIndex);
+      } catch {
+      }
+      for (const [, t] of this.tabs) {
+        if (t.tabIndex > oldest.tab.tabIndex) t.tabIndex--;
+      }
+    }
+    oldest.tab.active = false;
+    oldest.tab.url = void 0;
+    this.emitStatus(oldest.cid);
+    this.tabs.delete(oldest.cid);
+  }
+  /** Reconcile tab indices with the manager's actual page list. */
+  async reconcileTabs(mgr) {
+    try {
+      const managerTabs = await mgr.listTabs();
+      const managerUrls = managerTabs.map((t) => t.url);
+      for (const [cid, tab] of this.tabs) {
+        if (tab.tabIndex >= managerUrls.length) {
+          tab.active = false;
+          tab.url = void 0;
+          this.emitStatus(cid);
+          this.tabs.delete(cid);
+        }
+      }
+    } catch {
+    }
+  }
+  realTabCount() {
+    let n = 0;
+    for (const t of this.tabs.values()) {
+      if (t.tabIndex >= 0) n++;
+    }
+    return n;
+  }
+  async switchToConversation(mgr, conversationId) {
+    let tab = this.tabs.get(conversationId);
+    if (!tab || tab.tabIndex < 0) {
+      const realTabs = this.realTabCount();
+      if (realTabs >= MAX_TABS) {
+        await this.evictOldestTab(mgr);
+      }
+      if (realTabs > 0) {
+        await mgr.newTab();
+      }
+      const existing = tab;
+      tab = {
+        tabIndex: mgr.getActiveIndex(),
+        active: true,
+        lastUsed: Date.now(),
+        frameListeners: existing?.frameListeners ?? /* @__PURE__ */ new Set(),
+        statusListeners: existing?.statusListeners ?? /* @__PURE__ */ new Set()
+      };
+      this.tabs.set(conversationId, tab);
+    } else {
+      if (mgr.getActiveIndex() !== tab.tabIndex) {
+        await mgr.switchTo(tab.tabIndex);
+      }
+      tab.lastUsed = Date.now();
+    }
+    return tab;
+  }
+  /** Check if a conversation has an active browser tab. */
+  isActiveFor(conversationId) {
+    return this.tabs.has(conversationId) && this.tabs.get(conversationId).active;
+  }
+  /** Get the current URL for a conversation's tab. */
+  getUrl(conversationId) {
+    return this.tabs.get(conversationId)?.url;
+  }
+  /** Whether the browser has been launched. */
+  get isLaunched() {
+    return !!this.manager?.isLaunched();
+  }
+  // -----------------------------------------------------------------------
+  // Browser operations (all scoped by conversationId)
+  // -----------------------------------------------------------------------
+  async open(conversationId, url) {
+    await this.lock();
+    try {
+      return await this._doOpen(conversationId, url);
+    } catch (err) {
+      const msg = err?.message ?? "";
+      if (msg.includes("not launched") || msg.includes("closed") || msg.includes("Target closed")) {
+        console.log("[poncho][browser] Browser died mid-open, relaunching...");
+        try {
+          await this.manager?.close();
+        } catch {
+        }
+        this.manager = void 0;
+        for (const [, t] of this.tabs) {
+          if (t.tabIndex >= 0) {
+            t.tabIndex = -1;
+            t.active = false;
+            t.url = void 0;
+          }
+        }
+        return await this._doOpen(conversationId, url);
+      }
+      throw err;
+    } finally {
+      this.unlock();
+    }
+  }
+  async _doOpen(conversationId, url) {
+    const mgr = await this.ensureManager();
+    const tab = await this.switchToConversation(mgr, conversationId);
+    const page = mgr.getPage();
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
+    tab.url = page.url();
+    tab.active = true;
+    this.emitStatus(conversationId);
+    const title = await page.title();
+    return { title: title || void 0 };
+  }
+  async snapshot(conversationId) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const snap = await mgr.getSnapshot({ interactive: true, compact: true });
+      return snap.tree;
+    } finally {
+      this.unlock();
+    }
+  }
+  async click(conversationId, ref) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const locator = mgr.getLocatorFromRef(ref);
+      if (!locator) throw new Error(`No element found for ref ${ref}`);
+      await locator.click();
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+  async type(conversationId, ref, text) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const locator = mgr.getLocatorFromRef(ref);
+      if (!locator) throw new Error(`No element found for ref ${ref}`);
+      await locator.fill(text);
+      tab.url = mgr.getPage().url();
+    } finally {
+      this.unlock();
+    }
+  }
+  async screenshot(conversationId) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      const buf = await page.screenshot({ type: "jpeg", quality: 75 });
+      return buf.toString("base64");
+    } finally {
+      this.unlock();
+    }
+  }
+  async scroll(conversationId, direction, amount) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      const pixels = amount ?? 600;
+      const delta = direction === "down" ? pixels : -pixels;
+      await page.evaluate(`window.scrollBy(0, ${delta})`);
+    } finally {
+      this.unlock();
+    }
+  }
+  async closeTab(conversationId) {
+    await this.lock();
+    try {
+      const tab = this.tabs.get(conversationId);
+      if (!tab) return;
+      if (this._screencastConversation === conversationId) {
+        try {
+          await this.manager?.stopScreencast();
+        } catch {
+        }
+        this._screencastConversation = void 0;
+      }
+      const otherRealTabs = this.realTabCount() - (tab.tabIndex >= 0 ? 1 : 0);
+      if (otherRealTabs > 0 && this.manager?.isLaunched() && tab.tabIndex >= 0) {
+        try {
+          await this.manager.closeTab(tab.tabIndex);
+        } catch {
+        }
+        for (const [, t] of this.tabs) {
+          if (t.tabIndex > tab.tabIndex) t.tabIndex--;
+        }
+      } else if (this.manager?.isLaunched()) {
+        try {
+          await this.manager.close();
+        } catch {
+        }
+        this.manager = void 0;
+      }
+      tab.active = false;
+      tab.url = void 0;
+      this.emitStatus(conversationId);
+      this.tabs.delete(conversationId);
+    } finally {
+      this.unlock();
+    }
+  }
+  async navigate(conversationId, action) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = await this.switchToConversation(mgr, conversationId);
+      const page = mgr.getPage();
+      if (action === "back") await page.goBack();
+      else if (action === "forward") await page.goForward();
+      else throw new Error(`Unknown navigation action: ${action}`);
+      tab.url = page.url();
+    } finally {
+      this.unlock();
+    }
+  }
+  // -----------------------------------------------------------------------
+  // Screencast (one active at a time, tied to the viewed conversation)
+  // -----------------------------------------------------------------------
+  async startScreencast(conversationId, options) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      const tab = this.tabs.get(conversationId);
+      if (!tab) {
+        return;
+      }
+      if (mgr.isScreencasting()) {
+        try {
+          await mgr.stopScreencast();
+        } catch {
+        }
+      }
+      if (mgr.getActiveIndex() !== tab.tabIndex) {
+        await mgr.switchTo(tab.tabIndex);
+      }
+      this._screencastConversation = conversationId;
+      await mgr.startScreencast(
+        (frame) => {
+          const cid = this._screencastConversation;
+          if (!cid) return;
+          const t = this.tabs.get(cid);
+          if (!t) return;
+          const browserFrame = {
+            data: frame.data,
+            width: frame.metadata.deviceWidth,
+            height: frame.metadata.deviceHeight,
+            timestamp: Date.now()
+          };
+          for (const listener of t.frameListeners) {
+            try {
+              listener(browserFrame);
+            } catch {
+            }
+          }
+        },
+        {
+          format: options?.format ?? "jpeg",
+          quality: options?.quality ?? this.config.quality ?? 60,
+          maxWidth: options?.maxWidth ?? this.config.viewport?.width ?? 1280,
+          maxHeight: options?.maxHeight ?? this.config.viewport?.height ?? 720,
+          everyNthFrame: options?.everyNthFrame ?? this.config.everyNthFrame ?? 2
+        }
+      );
+    } finally {
+      this.unlock();
+    }
+  }
+  async stopScreencast() {
+    if (!this.manager?.isScreencasting()) return;
+    await this.manager.stopScreencast();
+    this._screencastConversation = void 0;
+  }
+  // -----------------------------------------------------------------------
+  // Per-conversation event listeners
+  // -----------------------------------------------------------------------
+  onFrame(conversationId, listener) {
+    let tab = this.tabs.get(conversationId);
+    if (!tab) {
+      tab = { tabIndex: -1, active: false, lastUsed: Date.now(), frameListeners: /* @__PURE__ */ new Set(), statusListeners: /* @__PURE__ */ new Set() };
+      this.tabs.set(conversationId, tab);
+    }
+    tab.frameListeners.add(listener);
+    return () => {
+      tab.frameListeners.delete(listener);
+    };
+  }
+  onStatus(conversationId, listener) {
+    let tab = this.tabs.get(conversationId);
+    if (!tab) {
+      tab = { tabIndex: -1, active: false, lastUsed: Date.now(), frameListeners: /* @__PURE__ */ new Set(), statusListeners: /* @__PURE__ */ new Set() };
+      this.tabs.set(conversationId, tab);
+    }
+    tab.statusListeners.add(listener);
+    return () => {
+      tab.statusListeners.delete(listener);
+    };
+  }
+  // -----------------------------------------------------------------------
+  // User input injection (all scoped by conversationId)
+  // -----------------------------------------------------------------------
+  async injectMouse(conversationId, event) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      await mgr.injectMouseEvent({
+        type: event.type,
+        x: event.x,
+        y: event.y,
+        button: event.button ?? "left",
+        clickCount: event.clickCount ?? 1,
+        deltaX: event.deltaX ?? 0,
+        deltaY: event.deltaY ?? 0
+      });
+    } finally {
+      this.unlock();
+    }
+  }
+  async injectKeyboard(conversationId, event) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const cdp = await mgr.getCDPSession();
+      let cdpType = event.type;
+      if (event.type === "keyDown" && !event.text) cdpType = "rawKeyDown";
+      await cdp.send("Input.dispatchKeyEvent", {
+        type: cdpType,
+        key: event.key,
+        code: event.code,
+        text: event.text,
+        windowsVirtualKeyCode: event.keyCode ?? 0,
+        nativeVirtualKeyCode: event.keyCode ?? 0
+      });
+    } finally {
+      this.unlock();
+    }
+  }
+  async injectPaste(conversationId, text) {
+    await this.lock();
+    try {
+      const mgr = await this.ensureManager();
+      await this.switchToConversation(mgr, conversationId);
+      const cdp = await mgr.getCDPSession();
+      await cdp.send("Input.insertText", { text });
+    } finally {
+      this.unlock();
+    }
+  }
+  async injectScroll(conversationId, event) {
+    await this.injectMouse(conversationId, {
+      type: "mouseWheel",
+      x: event.x ?? 0,
+      y: event.y ?? 0,
+      deltaX: event.deltaX,
+      deltaY: event.deltaY
+    });
+  }
+  // -----------------------------------------------------------------------
+  // Session persistence & shutdown
+  // -----------------------------------------------------------------------
+  async saveState(storagePath) {
+    if (!this.manager?.isLaunched()) return;
+    await mkdir(resolve(storagePath, ".."), { recursive: true });
+    await this.manager.saveStorageState(storagePath);
+  }
+  async close() {
+    try {
+      await this.stopScreencast();
+    } catch {
+    }
+    try {
+      await this.manager?.close();
+    } catch {
+    }
+    this.manager = void 0;
+    for (const [cid, tab] of this.tabs) {
+      tab.active = false;
+      tab.url = void 0;
+      this.emitStatus(cid);
+    }
+    this.tabs.clear();
+  }
+  // -----------------------------------------------------------------------
+  // Internals
+  // -----------------------------------------------------------------------
+  emitStatus(conversationId) {
+    const tab = this.tabs.get(conversationId);
+    const status = {
+      active: tab?.active ?? false,
+      url: tab?.url,
+      interactionAllowed: tab?.active ?? false
+    };
+    if (tab) {
+      for (const listener of tab.statusListeners) {
+        try {
+          listener(status);
+        } catch {
+        }
+      }
+    }
+  }
+};
+
+// src/tools.ts
+function createBrowserTools(getSession, getConversationId) {
+  return [
+    {
+      name: "browser_open",
+      description: "Open a URL in a headless browser. Returns the page title. Use this to navigate to websites and web applications.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          url: {
+            type: "string",
+            description: "The URL to navigate to (must include protocol, e.g. https://)"
+          }
+        },
+        required: ["url"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const cid = getConversationId();
+        const url = String(input.url ?? "");
+        if (!url) throw new Error("url is required");
+        const result = await session.open(cid, url);
+        session.startScreencast(cid).catch((err) => {
+          console.error("[poncho][browser] startScreencast failed:", err?.message ?? err);
+        });
+        return { url, title: result.title ?? "(no title)" };
+      }
+    },
+    {
+      name: "browser_snapshot",
+      description: "Get the current page as a compact accessibility tree with element refs (@e1, @e2, ...). Use refs to interact with elements via browser_click and browser_type. Re-snapshot after each interaction since refs change when the page updates.",
+      inputSchema: {
+        type: "object",
+        properties: {}
+      },
+      handler: async () => {
+        const session = getSession();
+        const snapshot = await session.snapshot(getConversationId());
+        return { snapshot };
+      }
+    },
+    {
+      name: "browser_click",
+      description: "Click an element identified by its ref from the last snapshot (e.g. @e2). Always take a snapshot first to get current refs.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          ref: {
+            type: "string",
+            description: 'Element ref from the snapshot (e.g. "@e2")'
+          }
+        },
+        required: ["ref"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const ref = String(input.ref ?? "");
+        if (!ref) throw new Error("ref is required");
+        await session.click(getConversationId(), ref);
+        return { clicked: ref };
+      }
+    },
+    {
+      name: "browser_type",
+      description: "Type text into a form field identified by its ref from the last snapshot. This clears the field first, then types the new value.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          ref: {
+            type: "string",
+            description: 'Element ref from the snapshot (e.g. "@e3")'
+          },
+          text: {
+            type: "string",
+            description: "Text to type into the field"
+          }
+        },
+        required: ["ref", "text"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const ref = String(input.ref ?? "");
+        const text = String(input.text ?? "");
+        if (!ref) throw new Error("ref is required");
+        await session.type(getConversationId(), ref, text);
+        return { typed: text, into: ref };
+      }
+    },
+    {
+      name: "browser_screenshot",
+      description: "Take a screenshot of the current page. Returns the image so you can see exactly what the page looks like. Use this when you need to see visual layout, verify actions, or read content that isn't in the accessibility tree.",
+      inputSchema: {
+        type: "object",
+        properties: {}
+      },
+      handler: async () => {
+        const session = getSession();
+        const base64 = await session.screenshot(getConversationId());
+        const filePart = {
+          type: "file",
+          data: base64,
+          mediaType: "image/jpeg",
+          filename: "screenshot.jpg"
+        };
+        return { screenshot: filePart };
+      }
+    },
+    {
+      name: "browser_scroll",
+      description: "Scroll the page up or down. Use this to see content that's below or above the current viewport.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          direction: {
+            type: "string",
+            enum: ["up", "down"],
+            description: "Scroll direction"
+          },
+          amount: {
+            type: "number",
+            description: "Pixels to scroll (default: one viewport height)"
+          }
+        },
+        required: ["direction"]
+      },
+      handler: async (input) => {
+        const session = getSession();
+        const direction = String(input.direction ?? "down");
+        const amount = typeof input.amount === "number" ? input.amount : void 0;
+        await session.scroll(getConversationId(), direction, amount);
+        return { scrolled: direction, amount: amount ?? "viewport" };
+      }
+    },
+    {
+      name: "browser_close",
+      description: "Close the browser tab for this conversation. Call this when you're done with browser tasks to free resources.",
+      inputSchema: {
+        type: "object",
+        properties: {}
+      },
+      handler: async () => {
+        const session = getSession();
+        await session.closeTab(getConversationId());
+        return { closed: true };
+      }
+    }
+  ];
+}
+export {
+  BrowserSession,
+  createBrowserTools
+};