@thoughtflow/browser 0.2.0 → 0.3.1

package/src/tools/workspace/browser.tool.ts

@@ -1,176 +1,375 @@
 import { z } from "zod";
-import { ToolBase, ToolContext, ToolExecutionResult } from "@thoughtflow/core";
+import { ToolBase, ToolContext, ToolExecutionResult, LlmAdapter } from "@thoughtflow/core";
+import { browserPool } from "./browser-session-pool";
+import { buildUserVisionMessage } from "@thoughtflow/core";
 
 const BROWSER_ACTIONS = [
+  "open",
   "navigate",
+  "snapshot",
   "get_text",
-  "screenshot",
   "get_dom",
-  "extract",
+  "screenshot",
   "click",
   "type",
   "select",
+  "scroll",
+  "console",
+  "network",
+  "vision",
+  "new_page",
+  "switch_page",
+  "list_pages",
+  "list",
+  "close_page",
   "close",
+  "close_all",
 ] as const;
 
 type BrowserAction = (typeof BROWSER_ACTIONS)[number];
 
-const READ_ONLY_BROWSER_ACTIONS = new Set<BrowserAction>([
-  "navigate",
-  "get_text",
-  "screenshot",
-  "get_dom",
-  "extract",
+const READ_ONLY_ACTIONS = new Set<BrowserAction>([
+  "snapshot", "get_text", "get_dom", "screenshot",
+  "console", "network", "list_pages", "list", "vision",
+]);
+
+const DESTRUCTIVE_ACTIONS = new Set<BrowserAction>([
+  "close", "close_page", "close_all",
 ]);
 
 /**
- * BrowserTool — Playwright-backed browser automation (Faza 8 stub).
+ * BrowserTool — stateful headless browser with session support.
  *
- * Playwright is loaded lazily on the first run() call so that registering
- * this tool does NOT require Playwright to be installed. If Playwright is
- * missing, run() returns `success: false` with a helpful install message.
+ * Sessions persist across calls via sessionId. Each session can have
+ * multiple tabs. Console and network activity is auto-captured.
  *
- * Note: Full implementation planned in Faza 8. Currently returns an
- * informative "not yet implemented" error for all actions.
+ * Use `open` to start, then navigate/interact/screenshot, finally `close`.
+ * Omit sessionId to use a default single session.
  */
 export class BrowserTool extends ToolBase {
   readonly name = "browser";
+  /** Static vision adapter — set once before use. Used by the `vision` action. */
+  static visionAdapter: LlmAdapter | null = null
+  static visionModel: string = "gemma3:12b"
   readonly description =
-    "Headless browser automation with Playwright (Chromium). " +
-    "Read: navigate, get_text, screenshot (vision only), get_dom, extract. " +
-    "Write: click, type, select, close. " +
-    "Playwright is loaded on first use — no install required at import time.";
+    "Stateful headless browser (Playwright/Chromium). " +
+    "Sessions persist across calls — open a session, navigate pages, " +
+    "interact, inspect, then close. Each session has its own browser process " +
+    "(isolated cookies/storage). Console and network auto-captured. " +
+    "Actions: open, navigate, snapshot (accessibility tree), get_text, " +
+    "get_dom (raw HTML), screenshot, click, type, select, scroll, " +
+    "console (browser logs), network (requests/responses), " +
+    "vision (visual AI analysis of current page), " +
+    "new_page, switch_page, list_pages, list (all sessions), " +
+    "close_page, close, close_all. " +
+    "Use snapshot/click with [ref=eN] selectors for reliable interaction. " +
+    "Sessions auto-expire after 5 minutes of inactivity.\n\n" +
+    "CANONICAL FLOW — follow this exactly:\n" +
+    "1. browser({action:\"open\"}) → save sessionId\n" +
+    "2. browser({action:\"navigate\", sessionId, url:\"...\"})\n" +
+    "3. browser({action:\"snapshot\", sessionId}) → find [ref=eN]\n" +
+    "4. browser({action:\"click\", sessionId, selector:\"e4\"})\n" +
+    "5. browser({action:\"vision\", sessionId, prompt:\"Is the button visible?\"})\n" +
+    "6. browser({action:\"console\", sessionId}) to check for JS errors\n" +
+    "7. browser({action:\"close\", sessionId}) when done\n" +
+    "ALWAYS reuse sessionId — NEVER call open again for the same browsing task."
 
-  /** Read-only actions: navigate/get_text/screenshot/get_dom/extract */
   override isReadOnly(input: Record<string, unknown>): boolean {
-    return READ_ONLY_BROWSER_ACTIONS.has((input["action"] as BrowserAction) ?? "navigate");
+    return READ_ONLY_ACTIONS.has((input["action"] as BrowserAction) ?? "navigate");
+  }
+
+  override isDestructive(input: Record<string, unknown>): boolean {
+    return DESTRUCTIVE_ACTIONS.has((input["action"] as BrowserAction) ?? "navigate");
   }
 
   constructor() {
     super(
       z.object({
-        action: z
-          .enum(BROWSER_ACTIONS)
-          .describe("Browser action to perform"),
-        url: z.string().optional().describe("URL to navigate to (for navigate action)"),
-        selector: z.string().optional().describe("CSS selector (for click/type/select/extract/get_dom)"),
-        text: z.string().optional().describe("Text to type (for type action)"),
-        value: z.string().optional().describe("Option value to select (for select action)"),
-        fields: z.array(z.string()).optional().describe("Fields to extract from rows (for extract action)"),
-        waitFor: z.number().optional().describe("Max ms to wait for page load (default: 5000)"),
+        action: z.enum(BROWSER_ACTIONS).describe("Browser action to perform"),
+        sessionId: z.string().optional()
+          .describe("Session ID from `open`. Omit for default single session."),
+        url: z.string().optional().describe("URL to navigate to"),
+        selector: z.string().optional()
+          .describe("CSS selector or [ref=eN] from snapshot"),
+        text: z.string().optional().describe("Text to type"),
+        value: z.string().optional().describe("Value to select or scroll amount"),
+        pageId: z.string().optional()
+          .describe("Tab/page ID for switch_page/close_page"),
+        clear: z.boolean().optional().default(false)
+          .describe("Clear console/network logs after reading"),
+        fullPage: z.boolean().optional().default(false)
+          .describe("Screenshot full page (not just viewport)"),
+        isolated: z.boolean().optional().default(true)
+          .describe("Use separate browser process for cookie/storage isolation (default: true)"),
+        prompt: z.string().optional()
+          .describe("Vision analysis prompt — what to check visually on the page"),
+        model: z.string().optional()
+          .describe("Vision model override (default: gemma3:12b)"),
+        useCached: z.boolean().optional().default(false)
+          .describe("Re-analyze the cached screenshot instead of taking a new one"),
       })
     );
   }
 
-  // Shared browser instance (reused across calls)
-  private browser: import("playwright").Browser | null = null;
-  private page: import("playwright").Page | null = null;
-
-  private async getPage(): Promise<import("playwright").Page> {
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    const { chromium } = require("playwright") as typeof import("playwright");
-    if (!this.browser || !this.browser.isConnected()) {
-      this.browser = await chromium.launch({ headless: true });
-    }
-    if (!this.page || this.page.isClosed()) {
-      this.page = await this.browser.newPage();
-    }
-    return this.page;
-  }
-
   async run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult> {
-    // Lazy-load Playwright on first use (optional peer dependency)
+    // Lazy-load Playwright
     try {
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
       require("playwright");
     } catch {
       return {
         success: false,
-        error:
-          "Playwright is not installed. Run: npm install playwright && npx playwright install chromium",
+        error: "Playwright is not installed. Run: npm install playwright && npx playwright install chromium",
       };
     }
 
     const action = (input["action"] as BrowserAction) ?? "navigate";
-    const waitFor = (input["waitFor"] as number | undefined) ?? 5000;
+    const sessionId = input["sessionId"] as string | undefined;
+    const clear = (input["clear"] as boolean) ?? false;
 
     try {
-      const page = await this.getPage();
-
       switch (action) {
+        case "open": {
+          const { sessionId: sid, page } = await browserPool.getOrCreate(sessionId);
+          return {
+            success: true,
+            output: JSON.stringify({
+              sessionId: sid,
+              message: "Browser session created. Use this sessionId for all subsequent calls.",
+            }),
+          };
+        }
+
         case "navigate": {
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
           const url = input["url"] as string;
-          if (!url) return { success: false, error: "url is required for navigate" };
-          await page.goto(url, { waitUntil: "domcontentloaded", timeout: waitFor });
+          if (!url) return { success: false, error: "url is required" };
+          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 15000 });
           const title = await page.title();
+          if (sessionId) browserPool.setInitUrl(sessionId, url);
           return { success: true, output: JSON.stringify({ title, url: page.url() }) };
         }
 
+        case "snapshot": {
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const tree = await this.getAccessibilityTree(page);
+          return { success: true, output: tree };
+        }
+
         case "get_text": {
-          const selector = (input["selector"] as string | undefined) ?? "body";
-          const text = await page.locator(selector).first().innerText({ timeout: waitFor });
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const sel = (input["selector"] as string) ?? "body";
+          const text = await page.locator(sel).first().innerText({ timeout: 5000 });
           return { success: true, output: text.trim() };
         }
 
         case "get_dom": {
-          const selector = (input["selector"] as string | undefined) ?? "body";
-          const html = await page.locator(selector).first().innerHTML({ timeout: waitFor });
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const sel = (input["selector"] as string) ?? "body";
+          const html = await page.locator(sel).first().innerHTML({ timeout: 5000 });
           return { success: true, output: html };
         }
 
-        case "extract": {
-          const selector = input["selector"] as string;
-          const fields = (input["fields"] as string[] | undefined) ?? [];
-          if (!selector) return { success: false, error: "selector is required for extract" };
-          const rows = await page.locator(selector).all();
-          const results: Record<string, string>[] = [];
-          for (const row of rows) {
-            if (fields.length === 0) {
-              results.push({ text: (await row.innerText()).trim() });
-            } else {
-              const obj: Record<string, string> = {};
-              for (const field of fields) {
-                const el = row.locator(`[class*="${field}"], [data-${field}], .${field}`).first();
-                obj[field] = (await el.innerText().catch(() => "")).trim();
-              }
-              results.push(obj);
-            }
-          }
-          return { success: true, output: JSON.stringify(results) };
+        case "screenshot": {
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const buf = await page.screenshot({
+            type: "png",
+            fullPage: (input["fullPage"] as boolean) ?? false,
+          });
+          const b64 = buf.toString("base64");
+          // Cache for vision action
+          if (sessionId) browserPool.cacheScreenshot(sessionId, b64);
+          return { success: true, output: `screenshot:base64:${b64}` };
         }
 
         case "click": {
-          const selector = input["selector"] as string;
-          if (!selector) return { success: false, error: "selector is required for click" };
-          await page.locator(selector).first().click({ timeout: waitFor });
-          return { success: true, output: `Clicked: ${selector}` };
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const sel = input["selector"] as string;
+          if (!sel) return { success: false, error: "selector is required" };
+          const locator = this.resolveSelector(page, sel);
+          await locator.first().click({ timeout: 5000 });
+          return { success: true, output: `Clicked: ${sel}` };
         }
 
         case "type": {
-          const selector = input["selector"] as string;
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const sel = input["selector"] as string;
           const text = input["text"] as string;
-          if (!selector || text === undefined) return { success: false, error: "selector and text required for type" };
-          await page.locator(selector).first().fill(text, { timeout: waitFor });
-          return { success: true, output: `Typed into ${selector}` };
+          if (!sel || text === undefined)
+            return { success: false, error: "selector and text required" };
+          const locator = this.resolveSelector(page, sel);
+          await locator.first().fill(text, { timeout: 5000 });
+          return { success: true, output: `Typed into ${sel}` };
         }
 
         case "select": {
-          const selector = input["selector"] as string;
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const sel = input["selector"] as string;
           const value = input["value"] as string;
-          if (!selector || !value) return { success: false, error: "selector and value required for select" };
-          await page.locator(selector).first().selectOption(value, { timeout: waitFor });
-          return { success: true, output: `Selected ${value} in ${selector}` };
+          if (!sel || !value)
+            return { success: false, error: "selector and value required" };
+          const locator = this.resolveSelector(page, sel);
+          await locator.first().selectOption(value, { timeout: 5000 });
+          return { success: true, output: `Selected ${value} in ${sel}` };
         }
 
-        case "screenshot": {
-          const buf = await page.screenshot({ type: "png" });
-          return { success: true, output: `screenshot:base64:${buf.toString("base64")}` };
+        case "scroll": {
+          const page = await this.requirePage(sessionId);
+          if (!page) return this.noSession();
+          const amount = parseInt(input["value"] as string ?? "") || 500;
+          await page.evaluate((px) => window.scrollBy(0, px), amount);
+          return { success: true, output: `Scrolled ${amount}px` };
+        }
+
+        case "vision": {
+          if (!BrowserTool.visionAdapter) {
+            return { success: false, error: "Vision adapter not configured. Set BrowserTool.visionAdapter before use." };
+          }
+          const prompt = input["prompt"] as string;
+          if (!prompt) return { success: false, error: "prompt is required for vision analysis" };
+          const useCached = (input["useCached"] as boolean) ?? false;
+          const model = input["model"] as string | undefined;
+
+          let base64Data: string | null = null;
+          if (useCached && sessionId) {
+            base64Data = browserPool.getCachedScreenshot(sessionId);
+          }
+          if (!base64Data) {
+            const page = await this.requirePage(sessionId);
+            if (!page) return this.noSession();
+            const buf = await page.screenshot({
+              type: "png",
+              fullPage: (input["fullPage"] as boolean) ?? false,
+            });
+            base64Data = buf.toString("base64");
+            if (sessionId) browserPool.cacheScreenshot(sessionId, base64Data);
+          }
+
+          const visionMessage = buildUserVisionMessage(
+            `You are a senior UI/UX designer and graphic design specialist. ` +
+            `Analyze this screenshot with extreme attention to visual detail:\n` +
+            `- Colors: exact hex/rgb values of backgrounds, text, borders, buttons\n` +
+            `- Borders: presence, color, width (px), radius, style (solid/dashed)\n` +
+            `- Spacing: padding, margins, gaps between elements (estimate px)\n` +
+            `- Typography: font sizes, weights, alignment, line heights\n` +
+            `- Layout: element positions, centering, whitespace balance\n` +
+            `- Visual bugs: overflow, misalignment, overlapping, cut-off text\n` +
+            `\n` +
+            `User request: ${prompt}\n` +
+            `\n` +
+            `Prioritize the user's request above these defaults. ` +
+            `If the user asks about something specific, focus on that ` +
+            `while still noting any obvious visual issues.`,
+            base64Data
+          );
+          const response = await BrowserTool.visionAdapter.chat({
+            model: model || BrowserTool.visionModel,
+            messages: [visionMessage],
+            tools: [],
+          });
+
+          return { success: true, output: response.message.content ?? "(no vision analysis)" };
+        }
+
+        case "console": {
+          if (!sessionId) return this.noSession();
+          const entries = browserPool.getConsole(sessionId);
+          if (clear) browserPool.clearConsole(sessionId);
+          if (entries.length === 0) return { success: true, output: "(no console output)" };
+          const lines = entries.map((e) => {
+            const tag = e.type === "error" ? "error" : e.type === "warning" ? "warn " : e.type;
+            const loc = e.location ? ` — ${e.location}` : "";
+            return `[${tag}] ${e.text.slice(0, 500)}${loc}`;
+          });
+          return { success: true, output: `Console (${entries.length} entries):\n${lines.join("\n")}` };
+        }
+
+        case "network": {
+          if (!sessionId) return this.noSession();
+          const entries = browserPool.getNetwork(sessionId);
+          if (clear) browserPool.clearNetwork(sessionId);
+          if (entries.length === 0) return { success: true, output: "(no network activity)" };
+          const sorted = [...entries].sort((a, b) => a.idx - b.idx);
+          const lines = sorted.map((e) => {
+            const status = e.error ? "FAILED" : `${e.status} ${e.statusText ?? ""}`;
+            const dur = e.duration != null ? ` (${e.duration}ms)` : "";
+            return `${e.error ? "✗" : "✓"} ${e.method} ${e.url.slice(0, 120)} → ${status}${dur}`;
+          });
+          return {
+            success: true,
+            output: `Network (${entries.length} requests):\n${lines.join("\n")}`,
+          };
+        }
+
+        case "new_page": {
+          if (!sessionId) return { success: false, error: "sessionId required — use `open` first" };
+          const { page } = await browserPool.newPage(sessionId);
+          return {
+            success: true,
+            output: JSON.stringify({ pageId: `pg_active`, url: page.url(), message: "New tab created" }),
+          };
+        }
+
+        case "switch_page": {
+          if (!sessionId) return this.noSession();
+          const pageId = input["pageId"] as string;
+          if (!pageId) return { success: false, error: "pageId required" };
+          const ok = browserPool.switchPage(sessionId, pageId);
+          return { success: ok, output: ok ? `Switched to ${pageId}` : `Page ${pageId} not found` };
+        }
+
+        case "list_pages": {
+          if (!sessionId) return this.noSession();
+          const pages = browserPool.listPages(sessionId);
+          const session = (browserPool as any).sessions?.get(sessionId);
+          if (session) {
+            for (const p of pages) {
+              const bp = session.pages?.find((bp2: any) => bp2.id === p.id && !bp2.page.isClosed());
+              if (bp) {
+                try { p.title = await bp.page.title(); } catch {}
+              }
+            }
+          }
+          return {
+            success: true,
+            output: JSON.stringify(pages.map(p => ({
+              id: p.id,
+              url: p.url,
+              title: p.title,
+              active: (p as any).active,
+            }))),
+          };
+        }
+
+        case "list": {
+          const sessions = browserPool.list();
+          if (sessions.length === 0) return { success: true, output: "(no active browser sessions)" };
+          return { success: true, output: JSON.stringify(sessions, null, 2) };
+        }
+
+        case "close_page": {
+          if (!sessionId) return this.noSession();
+          const pageId = input["pageId"] as string | undefined;
+          await browserPool.closePage(sessionId, pageId);
+          return { success: true, output: "Page closed" };
         }
 
         case "close": {
-          await this.page?.close();
-          this.page = null;
-          return { success: true, output: "Browser page closed" };
+          if (!sessionId) return { success: false, error: "sessionId required" };
+          await browserPool.destroy(sessionId);
+          return { success: true, output: `Session ${sessionId} closed` };
+        }
+
+        case "close_all": {
+          const count = await browserPool.destroyAll();
+          return { success: true, output: `Closed ${count} browser session(s)` };
         }
 
         default:
@@ -180,4 +379,70 @@ export class BrowserTool extends ToolBase {
       return { success: false, error: String(err instanceof Error ? err.message : err) };
     }
   }
-}
+
+  // ── Private helpers ────────────────────────────────────────────────────────
+
+  private async requirePage(sessionId?: string): Promise<any> {
+    const { page } = await browserPool.getOrCreate(sessionId);
+    return page;
+  }
+
+  private noSession(): ToolExecutionResult {
+    return { success: false, error: "No active session. Use action: 'open' first." };
+  }
+
+  /** Resolve [ref=eN] selectors from snapshot, or pass through CSS selectors. */
+  private resolveSelector(page: any, sel: string) {
+    const refMatch = sel.match(/^e(\d+)$/);
+    if (refMatch) {
+      return page.locator(`[data-ref="e${refMatch[1]}"]`);
+    }
+    const bracketMatch = sel.match(/^\[ref=e(\d+)\]$/);
+    if (bracketMatch) {
+      return page.locator(`[data-ref="e${bracketMatch[1]}"]`);
+    }
+    return page.locator(sel);
+  }
+
+  /** Build a simplified accessibility tree with [ref=eN] markers. */
+  private async getAccessibilityTree(page: any): Promise<string> {
+    const lines: string[] = [];
+
+    // Inject data-ref attributes into interactive elements
+    await page.evaluate(() => {
+      let counter = 0;
+      const interactive = document.querySelectorAll(
+        'a, button, input, select, textarea, [role="button"], [role="link"], [role="textbox"], [role="combobox"], [role="checkbox"], [role="radio"], [role="tab"], [role="menuitem"], [role="option"], [role="switch"], [onclick], summary, details'
+      );
+      interactive.forEach((el) => {
+        (el as HTMLElement).setAttribute("data-ref", `e${++counter}`);
+      });
+    });
+
+    // Use Playwright's accessibility snapshot
+    try {
+      const snapshot = await (page as any).accessibility.snapshot({ interestingOnly: false });
+      if (snapshot) {
+        this.flattenA11yTree(snapshot, lines, 0);
+      }
+    } catch {
+      const text = await page.locator("body").innerText();
+      lines.push(text.slice(0, 5000));
+    }
+
+    return lines.join("\n") || "(empty page)";
+  }
+
+  private flattenA11yTree(node: any, lines: string[], depth: number): void {
+    const indent = "  ".repeat(Math.min(depth, 6));
+    const role = node.role || "unknown";
+    const name = node.name ? ` "${node.name.slice(0, 80)}"` : "";
+    lines.push(`${indent}- ${role}${name}`);
+
+    if (node.children) {
+      for (const child of node.children) {
+        this.flattenA11yTree(child, lines, depth + 1);
+      }
+    }
+  }
+}