barebrowse 0.1.0 → 0.2.1

package/src/chromium.js

@@ -0,0 +1,148 @@
+/**
+ * chromium.js — Find, launch, and connect to Chromium-based browsers.
+ *
+ * Supports: Chrome, Chromium, Brave, Edge, Vivaldi, Arc, Opera.
+ * Modes: headless (launch new), headed (connect to running).
+ */
+
+import { execSync, spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+
+// Common Chromium binary paths by platform (Linux focus for POC)
+const CANDIDATES = [
+  // Linux
+  'chromium-browser',
+  'chromium',
+  'google-chrome-stable',
+  'google-chrome',
+  'brave-browser-stable',
+  'brave-browser',
+  'microsoft-edge-stable',
+  'microsoft-edge',
+  'vivaldi-stable',
+  'vivaldi',
+  // macOS (future)
+  '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+  '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
+  '/Applications/Chromium.app/Contents/MacOS/Chromium',
+  '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+];
+
+/**
+ * Find the first available Chromium binary on the system.
+ * @returns {string} Path to the binary
+ * @throws {Error} If no Chromium browser is found
+ */
+export function findBrowser() {
+  for (const candidate of CANDIDATES) {
+    try {
+      // Absolute path — check directly
+      if (candidate.startsWith('/')) {
+        if (existsSync(candidate)) return candidate;
+        continue;
+      }
+      // Relative name — check via which
+      const path = execSync(`which ${candidate} 2>/dev/null`, { encoding: 'utf8' }).trim();
+      if (path) return path;
+    } catch {
+      // Not found, try next
+    }
+  }
+  throw new Error(
+    'No Chromium-based browser found. Install Chrome, Chromium, Brave, or Edge.\n' +
+    'On Fedora: sudo dnf install chromium'
+  );
+}
+
+/**
+ * Launch a headless Chromium instance with CDP enabled.
+ * @param {object} [opts]
+ * @param {string} [opts.binary] - Path to browser binary (auto-detected if omitted)
+ * @param {number} [opts.port=0] - CDP port (0 = random available port)
+ * @param {string} [opts.userDataDir] - Browser profile directory
+ * @returns {Promise<{wsUrl: string, process: ChildProcess, port: number}>}
+ */
+export async function launch(opts = {}) {
+  const binary = opts.binary || findBrowser();
+  const port = opts.port || 0;
+
+  const args = [
+    '--headless=new',
+    `--remote-debugging-port=${port}`,
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-background-networking',
+    '--disable-sync',
+    '--disable-translate',
+    '--mute-audio',
+    '--hide-scrollbars',
+    // Suppress permission prompts (location, notifications, camera, mic, etc.)
+    '--disable-notifications',
+    '--autoplay-policy=no-user-gesture-required',
+    '--use-fake-device-for-media-stream',
+    '--use-fake-ui-for-media-stream',
+    '--disable-features=MediaRouter',
+  ];
+
+  if (opts.userDataDir) {
+    args.push(`--user-data-dir=${opts.userDataDir}`);
+  } else {
+    // Use a unique temp profile so we don't lock the user's profile
+    // or conflict with parallel instances
+    args.push(`--user-data-dir=/tmp/barebrowse-${process.pid}-${Date.now()}`);
+  }
+
+  // about:blank as initial page
+  args.push('about:blank');
+
+  const child = spawn(binary, args, {
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+
+  // Parse the WebSocket URL from stderr
+  // Chrome prints: "DevTools listening on ws://127.0.0.1:PORT/devtools/browser/UUID"
+  const wsUrl = await new Promise((resolve, reject) => {
+    let stderr = '';
+    const timeout = setTimeout(() => {
+      reject(new Error(`Browser failed to start within 10s. stderr: ${stderr}`));
+    }, 10000);
+
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+      const match = stderr.match(/ws:\/\/[^\s]+/);
+      if (match) {
+        clearTimeout(timeout);
+        resolve(match[0]);
+      }
+    });
+
+    child.on('error', (err) => {
+      clearTimeout(timeout);
+      reject(new Error(`Failed to launch browser: ${err.message}`));
+    });
+
+    child.on('exit', (code) => {
+      clearTimeout(timeout);
+      if (!stderr.includes('ws://')) {
+        reject(new Error(`Browser exited with code ${code}. stderr: ${stderr}`));
+      }
+    });
+  });
+
+  // Extract port from wsUrl
+  const actualPort = parseInt(new URL(wsUrl).port, 10);
+
+  return { wsUrl, process: child, port: actualPort };
+}
+
+/**
+ * Get the CDP WebSocket URL for a browser already running with --remote-debugging-port.
+ * @param {number} port - The debug port
+ * @returns {Promise<string>} WebSocket URL
+ */
+export async function getDebugUrl(port) {
+  const res = await fetch(`http://127.0.0.1:${port}/json/version`);
+  if (!res.ok) throw new Error(`Cannot reach browser debug port at ${port}: ${res.status}`);
+  const data = await res.json();
+  return data.webSocketDebuggerUrl;
+}

package/src/consent.js

@@ -0,0 +1,210 @@
+/**
+ * consent.js — Auto-dismiss cookie consent dialogs via ARIA tree inspection.
+ *
+ * Scans the page ARIA tree for consent dialogs and clicks the "accept" button.
+ * Works across languages by matching common accept/agree patterns.
+ * Runs once after page load — no polling, no mutation observers.
+ */
+
+// Button text patterns that mean "accept all" / "I agree" across common languages.
+// Order matters: more specific patterns first to avoid false positives.
+const ACCEPT_PATTERNS = [
+  // English
+  /\baccept\s*all\b/i,
+  /\ballow\s*all\b/i,
+  /\bagree\s*to\s*all\b/i,
+  /\byes,?\s*i\s*agree\b/i,
+  /\bi\s*agree\b/i,
+  /\baccept\s*cookies?\b/i,
+  /\ballow\s*cookies?\b/i,
+  /\bgot\s*it\b/i,
+  // Dutch
+  /\balles\s*accepteren\b/i,
+  /\balles\s*toestaan\b/i,
+  /\bakkoord\b/i,
+  // German
+  /\balle\s*akzeptieren\b/i,
+  /\ballem\s*zustimmen\b/i,
+  /\balle\s*cookies?\s*akzeptieren\b/i,
+  // French
+  /\btout\s*accepter\b/i,
+  /\baccepter\s*tout\b/i,
+  /\bj['']accepte\b/i,
+  // Spanish
+  /\baceptar\s*todo\b/i,
+  /\baceptar\s*todas?\b/i,
+  // Italian
+  /\baccetta\s*tutto\b/i,
+  /\baccetto\b/i,
+  // Portuguese
+  /\baceitar\s*tudo\b/i,
+  // Generic single-word fallbacks (only matched inside dialogs)
+  /^accept$/i,
+  /^agree$/i,
+  /^ok$/i,
+];
+
+// Roles that indicate a consent dialog container.
+const DIALOG_ROLES = new Set(['dialog', 'alertdialog']);
+
+// Text patterns in dialog names/headings that confirm it's about consent.
+const CONSENT_DIALOG_HINTS = [
+  /cookie/i,
+  /consent/i,
+  /privacy/i,
+  /voordat\s*je\s*verdergaat/i,  // Dutch: "Before you continue"
+  /before\s*you\s*continue/i,
+  /bevor\s*du\s*fortf/i,         // German: "Before you continue"
+  /avant\s*de\s*continuer/i,     // French: "Before you continue"
+];
+
+/**
+ * Click a node via JavaScript .click() instead of mouse events.
+ * Bypasses iframe overlays and z-index issues that block coordinate-based clicks.
+ */
+async function jsClick(session, backendNodeId) {
+  const { object } = await session.send('DOM.resolveNode', { backendNodeId });
+  await session.send('Runtime.callFunctionOn', {
+    objectId: object.objectId,
+    functionDeclaration: 'function() { this.click(); }',
+  });
+}
+
+/**
+ * Try to dismiss a cookie consent dialog on the current page.
+ * Inspects the ARIA tree for dialog elements with consent-related content,
+ * then clicks the "accept" button.
+ *
+ * @param {object} session - Session-scoped CDP handle
+ * @returns {Promise<boolean>} true if a consent dialog was dismissed
+ */
+export async function dismissConsent(session) {
+  await session.send('Accessibility.enable');
+  const { nodes } = await session.send('Accessibility.getFullAXTree');
+
+  // Build a parent lookup: nodeId → parentId
+  const parentMap = new Map();
+  const nodeMap = new Map();
+  for (const node of nodes) {
+    nodeMap.set(node.nodeId, node);
+    if (node.parentId) parentMap.set(node.nodeId, node.parentId);
+  }
+
+  // Find dialog nodes that look like consent dialogs
+  const consentDialogs = new Set();
+  for (const node of nodes) {
+    const role = node.role?.value;
+    if (!DIALOG_ROLES.has(role)) continue;
+
+    const name = node.name?.value || '';
+    // Check if dialog name hints at consent
+    if (CONSENT_DIALOG_HINTS.some((p) => p.test(name))) {
+      consentDialogs.add(node.nodeId);
+      continue;
+    }
+
+    // Check children for consent-related headings/text
+    if (hasConsentContent(node.nodeId, nodes, nodeMap, parentMap)) {
+      consentDialogs.add(node.nodeId);
+    }
+  }
+
+  // If no consent dialog found, scan for consent-related buttons anywhere
+  // (some sites use banners, not dialogs)
+  if (consentDialogs.size === 0) {
+    return tryGlobalConsentButton(nodes, session);
+  }
+
+  // Find accept buttons inside consent dialogs
+  for (const dialogId of consentDialogs) {
+    const button = findAcceptButton(dialogId, nodes, nodeMap, parentMap);
+    if (button?.backendDOMNodeId) {
+      try {
+        await jsClick(session, button.backendDOMNodeId);
+        await new Promise((r) => setTimeout(r, 1000));
+        return true;
+      } catch {
+        // Click failed — try next dialog
+      }
+    }
+  }
+
+  // Dialog found but no accept button inside it — some sites put the button
+  // outside the dialog (e.g. BBC's SourcePoint). Fall through to global scan.
+  return tryGlobalConsentButton(nodes, session);
+}
+
+/**
+ * Check if a dialog contains consent-related content in its descendants.
+ */
+function hasConsentContent(dialogId, nodes, nodeMap, parentMap) {
+  for (const node of nodes) {
+    if (!isDescendantOf(node.nodeId, dialogId, parentMap)) continue;
+    const role = node.role?.value;
+    const name = node.name?.value || '';
+
+    // Check headings and static text within the dialog
+    if (role === 'heading' || role === 'StaticText' || role === 'generic') {
+      if (CONSENT_DIALOG_HINTS.some((p) => p.test(name))) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Find the best "accept" button inside a dialog subtree.
+ */
+function findAcceptButton(dialogId, nodes, nodeMap, parentMap) {
+  for (const pattern of ACCEPT_PATTERNS) {
+    for (const node of nodes) {
+      if (node.role?.value !== 'button') continue;
+      const name = node.name?.value || '';
+      if (!name || !pattern.test(name)) continue;
+      if (!isDescendantOf(node.nodeId, dialogId, parentMap)) continue;
+      return node;
+    }
+  }
+  return null;
+}
+
+/**
+ * Fallback: look for consent buttons anywhere on the page.
+ * Only matches strong patterns (not single-word fallbacks) to avoid false positives.
+ */
+function tryGlobalConsentButton(nodes, session) {
+  // Only use the specific multi-word patterns for global search
+  const strictPatterns = ACCEPT_PATTERNS.filter((p) => {
+    const src = p.source;
+    return src.includes('\\s') || src.includes('\\b.*\\b.*\\b');
+  });
+
+  // Actually, let's just use all non-single-word patterns
+  const safePatterns = ACCEPT_PATTERNS.slice(0, -3); // exclude ^accept$, ^agree$, ^ok$
+
+  for (const pattern of safePatterns) {
+    for (const node of nodes) {
+      if (node.role?.value !== 'button') continue;
+      const name = node.name?.value || '';
+      if (name && pattern.test(name) && node.backendDOMNodeId) {
+        return jsClick(session, node.backendDOMNodeId)
+          .then(() => new Promise((r) => setTimeout(r, 1000)))
+          .then(() => true)
+          .catch(() => false);
+      }
+    }
+  }
+
+  return Promise.resolve(false);
+}
+
+/**
+ * Check if nodeId is a descendant of ancestorId by walking parentMap.
+ */
+function isDescendantOf(nodeId, ancestorId, parentMap) {
+  let current = parentMap.get(nodeId);
+  while (current) {
+    if (current === ancestorId) return true;
+    current = parentMap.get(current);
+  }
+  return false;
+}

package/src/index.js

@@ -13,7 +13,9 @@ import { createCDP } from './cdp.js';
 import { formatTree } from './aria.js';
 import { authenticate } from './auth.js';
 import { prune as pruneTree } from './prune.js';
-import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress } from './interact.js';
+import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect } from './interact.js';
+import { dismissConsent } from './consent.js';
+import { applyStealth } from './stealth.js';
 
 /**
  * Browse a URL and return an ARIA snapshot.
@@ -42,13 +44,16 @@ export async function browse(url, opts = {}) {
       const wsUrl = await getDebugUrl(port);
       cdp = await createCDP(wsUrl);
     } else {
-      // headless (hybrid fallback logic comes in Phase 4)
+      // headless or hybrid (start headless)
       browser = await launch();
       cdp = await createCDP(browser.wsUrl);
     }
 
     // Step 2: Create a new page target and attach
-    const page = await createPage(cdp);
+    let page = await createPage(cdp, mode !== 'headed');
+
+    // Step 2.5: Suppress permission prompts
+    await suppressPermissions(cdp);
 
     // Step 3: Cookie injection — extract from user's browser, inject via CDP
     if (opts.cookies !== false) {
@@ -62,8 +67,32 @@ export async function browse(url, opts = {}) {
     // Step 4: Navigate and wait for load
     await navigate(page, url, timeout);
 
+    // Step 4.5: Auto-dismiss cookie consent dialogs
+    if (opts.consent !== false) {
+      await dismissConsent(page.session);
+    }
+
     // Step 5: Get ARIA tree
-    const { tree } = await ariaTree(page);
+    let { tree } = await ariaTree(page);
+
+    // Step 5.5: Hybrid fallback — if headless was bot-blocked, retry headed
+    if (mode === 'hybrid' && isChallengePage(tree)) {
+      await cdp.send('Target.closeTarget', { targetId: page.targetId });
+      cdp.close();
+      if (browser) { browser.process.kill(); browser = null; }
+
+      const port = opts.port || 9222;
+      const wsUrl = await getDebugUrl(port);
+      cdp = await createCDP(wsUrl);
+      page = await createPage(cdp, false);
+      await suppressPermissions(cdp);
+      if (opts.cookies !== false) {
+        try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
+      }
+      await navigate(page, url, timeout);
+      if (opts.consent !== false) await dismissConsent(page.session);
+      ({ tree } = await ariaTree(page));
+    }
 
     // Step 6: Prune for agent consumption
     let snapshot;
@@ -106,12 +135,22 @@ export async function connect(opts = {}) {
     cdp = await createCDP(browser.wsUrl);
   }
 
-  const page = await createPage(cdp);
+  const page = await createPage(cdp, mode !== 'headed');
   let refMap = new Map();
 
+  // Suppress permission prompts for all modes
+  await suppressPermissions(cdp);
+
   return {
     async goto(url, timeout = 30000) {
       await navigate(page, url, timeout);
+      if (opts.consent !== false) {
+        await dismissConsent(page.session);
+      }
+    },
+
+    async injectCookies(url, cookieOpts) {
+      await authenticate(page.session, url, { browser: cookieOpts?.browser });
     },
 
     async snapshot(pruneOpts) {
@@ -128,10 +167,10 @@ export async function connect(opts = {}) {
       await cdpClick(page.session, backendNodeId);
     },
 
-    async type(ref, text, opts) {
+    async type(ref, text, typeOpts) {
       const backendNodeId = refMap.get(ref);
       if (!backendNodeId) throw new Error(`No element found for ref "${ref}"`);
-      await cdpType(page.session, backendNodeId, text, opts);
+      await cdpType(page.session, backendNodeId, text, typeOpts);
     },
 
     async scroll(deltaY) {
@@ -142,8 +181,42 @@ export async function connect(opts = {}) {
       await cdpPress(page.session, key);
     },
 
-    waitForNavigation(timeout = 30000) {
-      return page.session.once('Page.loadEventFired', timeout);
+    async hover(ref) {
+      const backendNodeId = refMap.get(ref);
+      if (!backendNodeId) throw new Error(`No element found for ref "${ref}"`);
+      await cdpHover(page.session, backendNodeId);
+    },
+
+    async select(ref, value) {
+      const backendNodeId = refMap.get(ref);
+      if (!backendNodeId) throw new Error(`No element found for ref "${ref}"`);
+      await cdpSelect(page.session, backendNodeId, value);
+    },
+
+    async screenshot(screenshotOpts = {}) {
+      const format = screenshotOpts.format || 'png';
+      const params = { format };
+      if (format === 'jpeg' || format === 'webp') {
+        params.quality = screenshotOpts.quality || 80;
+      }
+      const { data } = await page.session.send('Page.captureScreenshot', params);
+      return data;
+    },
+
+    async waitForNavigation(timeout = 30000) {
+      // Wait for loadEventFired (full page load). If it doesn't fire within
+      // timeout, fall back to frameNavigated (SPA pushState/replaceState).
+      try {
+        await page.session.once('Page.loadEventFired', timeout);
+      } catch {
+        // Timeout — likely SPA nav with no load event. frameNavigated may
+        // have already fired. Give a settle delay for DOM updates.
+        await new Promise((r) => setTimeout(r, 500));
+      }
+    },
+
+    waitForNetworkIdle(idleOpts = {}) {
+      return waitForNetworkIdle(page.session, idleOpts);
     },
 
     /** Raw CDP session for escape hatch */
@@ -159,10 +232,35 @@ export async function connect(opts = {}) {
 
 // --- Internal helpers ---
 
+/**
+ * Suppress permission prompts (notifications, geolocation, camera, mic, etc.)
+ * via CDP Browser.setPermission. Works for both headless and headed modes.
+ */
+const DENY_PERMISSIONS = [
+  'geolocation', 'notifications', 'midi', 'midiSysex',
+  'durableStorage', 'audioCapture', 'videoCapture',
+  'backgroundSync', 'sensors', 'idleDetection',
+];
+
+async function suppressPermissions(cdp) {
+  for (const name of DENY_PERMISSIONS) {
+    try {
+      await cdp.send('Browser.setPermission', {
+        permission: { name },
+        setting: 'denied',
+      });
+    } catch {
+      // Permission type not supported in this Chrome version — skip
+    }
+  }
+}
+
 /**
  * Create a new page target and return a session-scoped handle.
+ * @param {object} cdp - CDP client
+ * @param {boolean} [stealth=false] - Apply stealth patches (headless only)
  */
-async function createPage(cdp) {
+async function createPage(cdp, stealth = false) {
   const { targetId } = await cdp.send('Target.createTarget', { url: 'about:blank' });
   const { sessionId } = await cdp.send('Target.attachToTarget', {
     targetId,
@@ -176,6 +274,11 @@ async function createPage(cdp) {
   await session.send('Network.enable');
   await session.send('DOM.enable');
 
+  // Apply stealth patches before any navigation (headless only)
+  if (stealth) {
+    await applyStealth(session);
+  }
+
   return { session, targetId, sessionId };
 }
 
@@ -256,3 +359,76 @@ function extractProps(props) {
   for (const p of props) result[p.name] = p.value?.value;
   return result;
 }
+
+/**
+ * Wait until no network requests are pending for `idle` ms.
+ * @param {object} session - Session-scoped CDP handle
+ * @param {object} [opts]
+ * @param {number} [opts.timeout=30000] - Max wait time
+ * @param {number} [opts.idle=500] - Idle threshold in ms
+ */
+function waitForNetworkIdle(session, opts = {}) {
+  const timeout = opts.timeout || 30000;
+  const idle = opts.idle || 500;
+
+  return new Promise((resolve, reject) => {
+    let pending = 0;
+    let timer = null;
+    const unsubs = [];
+
+    const done = () => {
+      clearTimeout(timer);
+      clearTimeout(deadlineTimer);
+      for (const unsub of unsubs) unsub();
+      resolve();
+    };
+
+    const check = () => {
+      clearTimeout(timer);
+      if (pending <= 0) {
+        pending = 0;
+        timer = setTimeout(done, idle);
+      }
+    };
+
+    unsubs.push(session.on('Network.requestWillBeSent', () => { pending++; clearTimeout(timer); }));
+    unsubs.push(session.on('Network.loadingFinished', () => { pending--; check(); }));
+    unsubs.push(session.on('Network.loadingFailed', () => { pending--; check(); }));
+
+    const deadlineTimer = setTimeout(() => {
+      for (const unsub of unsubs) unsub();
+      reject(new Error(`waitForNetworkIdle timed out after ${timeout}ms`));
+    }, timeout);
+
+    // Start check immediately (might already be idle)
+    check();
+  });
+}
+
+/**
+ * Detect if a page is a bot-challenge page (Cloudflare, etc.).
+ * Heuristic: very short ARIA tree + known challenge phrases.
+ */
+function isChallengePage(tree) {
+  if (!tree) return true;
+  const text = flattenTreeText(tree);
+  const challengePhrases = [
+    'just a moment',
+    'checking if the site connection is secure',
+    'checking your browser',
+    'please wait',
+    'verify you are human',
+    'attention required',
+  ];
+  const lower = text.toLowerCase();
+  return challengePhrases.some((p) => lower.includes(p));
+}
+
+function flattenTreeText(node) {
+  if (!node) return '';
+  let text = node.name || '';
+  for (const child of node.children || []) {
+    text += ' ' + flattenTreeText(child);
+  }
+  return text;
+}