barebrowse 0.5.6 → 0.5.8

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.5.8
+
+Bot challenge detection for all browsing, not just assess.
+
+### Bot detection (`src/index.js`)
+- `isChallengePage()` now checks ARIA node count (<50 = bot-blocked) in addition to text length and phrase matching
+- `botBlocked` property exposed on both `connect()` pages and `createTab()` tabs
+- `goto()` on main page and tabs sets `botBlocked` after every navigation
+- `snapshot()` prepends `[BOT CHALLENGE DETECTED]` warning line when flagged
+- Hybrid fallback on main page now uses node count for more reliable detection
+
+### Assess handler (`mcp-server.js`)
+- Headed fallback now uses `tab.botBlocked` flag instead of naive score threshold (≤5 + all zeros)
+- Previously: sites like Reuters, Home Depot, Leboncoin returned fake-clean scores because the bot challenge page looked "clean" to the scanner
+- Now: node count catches every bot-blocked page regardless of score
+
+### Tested
+- reuters.com, homedepot.com, leboncoin.fr, idealista.com all correctly flagged `botBlocked: true`
+- svt.se, whatsapp.com, google.com correctly flagged `false`
+- 71/71 tests passing
+
+## 0.5.7
+
+MCP server crash resilience + process hardening.
+
+### Process hardening (`mcp-server.js`)
+- Added `unhandledRejection` and `uncaughtException` handlers — browser OOM/crash no longer kills the MCP server process
+- Previously: heavy sites like zalando.de crashed the browser via OOM, the CDP WebSocket close rejected pending promises, unhandled rejections crashed Node
+- Now: session resets and next request gets a fresh browser. Server stays alive.
+
+### Validated at scale
+- Scanned 149 sites across NL/US/EU with zero server crashes
+- Only 1 genuine timeout (rtv.nl) — all former crashers (zalando.de, otto.de, bijenkorf.nl, jumbo.com, klm.nl) now return results
+- Full results: `wearehere-scan-results.md`
+
 ## 0.5.6
 
 Assess now works on bot-blocking EU sites. Headed fallback + consent fix.

package/barebrowse.context.md

@@ -1,7 +1,7 @@
 # barebrowse -- Integration Guide
 
 > For AI assistants and developers wiring barebrowse into a project.
-> v0.5.6 | Node.js >= 22 | 0 required deps | MIT
+> v0.5.8 | Node.js >= 22 | 0 required deps | MIT
 
 ## What this is
 
@@ -243,7 +243,7 @@ Action tools return `'ok'` -- the agent calls `snapshot` explicitly to observe.
 
 Session runs in hybrid mode (headless with automatic headed fallback on bot detection). `goto` injects cookies from the user's browser before navigation for authenticated access.
 
-Session tools share a singleton page, lazy-created on first use. Assess tries headless first; if bot-blocked (score ≤5 with all zeros), retries with a separate headed session. Tabs dismissed for consent and closed after every scan. Max 3 concurrent, with CDP crash recovery.
+Session tools share a singleton page, lazy-created on first use. Assess tries headless first; if bot-blocked (score ≤5 with all zeros), retries with a separate headed session. Tabs dismissed for consent and closed after every scan. Max 3 concurrent. Browser OOM/crash auto-recovers (session resets, server stays alive).
 
 ## Architecture
 

package/mcp-server.js

@@ -317,8 +317,9 @@ async function handleToolCall(name, args) {
               }),
             ]);
             clearTimeout(timer);
+            const wasBotBlocked = tab.botBlocked;
             await tab.close().catch(() => {});
-            return JSON.stringify(result, null, 2);
+            return { result, botBlocked: wasBotBlocked };
           } catch (err) {
             clearTimeout(timer);
             await tab.close().catch(() => {});
@@ -328,29 +329,23 @@ async function handleToolCall(name, args) {
 
         // Try headless first
         try {
-          const result = await runAssess(false);
-          // Check if result looks bot-blocked (score 0-5, no trackers, few cookies)
-          try {
-            const parsed = JSON.parse(result);
-            const { network, trackers, profiling } = parsed.categories || {};
-            const allZero = (network?.score || 0) === 0
-              && (trackers?.score || 0) === 0
-              && (profiling?.score || 0) === 0;
-            if (allZero && (parsed.score || 0) <= 5) {
-              // Likely bot-blocked — retry headed
-              try {
-                return await runAssess(true);
-              } catch {
-                return result; // headed failed, return headless result
-              }
+          const { result, botBlocked } = await runAssess(false);
+          if (botBlocked) {
+            // Bot-blocked in headless — retry headed
+            try {
+              const headed = await runAssess(true);
+              return JSON.stringify(headed.result, null, 2);
+            } catch {
+              return JSON.stringify(result, null, 2); // headed failed, return headless result
             }
-          } catch {}
-          return result;
+          }
+          return JSON.stringify(result, null, 2);
         } catch (err) {
           if (isCdpDead(err)) _page = null;
           // Headless crashed — try headed
           try {
-            return await runAssess(true);
+            const headed = await runAssess(true);
+            return JSON.stringify(headed.result, null, 2);
           } catch (retryErr) {
             throw retryErr;
           }
@@ -379,7 +374,7 @@ async function handleMessage(msg) {
     return jsonrpcResponse(id, {
       protocolVersion: '2024-11-05',
       capabilities: { tools: {} },
-      serverInfo: { name: 'barebrowse', version: '0.5.6' },
+      serverInfo: { name: 'barebrowse', version: '0.5.8' },
     });
   }
 
@@ -443,6 +438,15 @@ process.stdin.on('data', (chunk) => {
   }
 });
 
+// Prevent unhandled rejections and uncaught exceptions from crashing the server.
+// Browser OOM/crash rejects all pending CDP promises — some may not be awaited.
+process.on('unhandledRejection', (err) => {
+  _page = null;
+});
+process.on('uncaughtException', (err) => {
+  _page = null;
+});
+
 // Clean up on exit
 process.on('SIGINT', async () => {
   if (_page) await _page.close().catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "barebrowse",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "description": "Authenticated web browsing for autonomous agents via CDP. URL in, pruned ARIA snapshot out.",
   "type": "module",
   "main": "src/index.js",

package/src/index.js

@@ -73,10 +73,10 @@ export async function browse(url, opts = {}) {
     }
 
     // Step 5: Get ARIA tree
-    let { tree } = await ariaTree(page);
+    let { tree, nodeCount } = await ariaTree(page);
 
     // Step 5.5: Hybrid fallback — if headless was bot-blocked, retry headed
-    if (mode === 'hybrid' && isChallengePage(tree)) {
+    if (mode === 'hybrid' && isChallengePage(tree, nodeCount)) {
       await cdp.send('Target.closeTarget', { targetId: page.targetId });
       cdp.close();
       if (browser) { browser.process.kill(); browser = null; }
@@ -140,6 +140,7 @@ export async function connect(opts = {}) {
 
   let page = await createPage(cdp, mode !== 'headed', { viewport: opts.viewport });
   let refMap = new Map();
+  let botBlocked = false;
 
   // Suppress permission prompts for all modes
   await suppressPermissions(cdp);
@@ -179,23 +180,28 @@ export async function connect(opts = {}) {
         await dismissConsent(page.session);
       }
 
+      // Check for bot challenge
+      const { tree, nodeCount } = await ariaTree(page);
+      botBlocked = isChallengePage(tree, nodeCount);
+
       // Hybrid fallback: if bot-blocked, retry with headed browser
-      if (mode === 'hybrid') {
-        const { tree } = await ariaTree(page);
-        if (isChallengePage(tree)) {
-          await cdp.send('Target.closeTarget', { targetId: page.targetId });
-          cdp.close();
-          if (browser) { browser.process.kill(); browser = null; }
-
-          const port = opts.port || 9222;
-          const wsUrl = await getDebugUrl(port);
-          cdp = await createCDP(wsUrl);
-          page = await createPage(cdp, false, { viewport: opts.viewport });
-          setupDialogHandler(page.session);
-          await suppressPermissions(cdp);
-          await navigate(page, url, timeout);
-          if (opts.consent !== false) await dismissConsent(page.session);
-        }
+      if (botBlocked && mode === 'hybrid') {
+        await cdp.send('Target.closeTarget', { targetId: page.targetId });
+        cdp.close();
+        if (browser) { browser.process.kill(); browser = null; }
+
+        const port = opts.port || 9222;
+        const wsUrl = await getDebugUrl(port);
+        cdp = await createCDP(wsUrl);
+        page = await createPage(cdp, false, { viewport: opts.viewport });
+        setupDialogHandler(page.session);
+        await suppressPermissions(cdp);
+        await navigate(page, url, timeout);
+        if (opts.consent !== false) await dismissConsent(page.session);
+
+        // Re-check after headed fallback
+        const after = await ariaTree(page);
+        botBlocked = isChallengePage(after.tree, after.nodeCount);
       }
     },
 
@@ -223,11 +229,12 @@ export async function connect(opts = {}) {
       const raw = formatTree(result.tree);
       const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
       const pageUrl = entries[currentIndex]?.url || '';
-      if (pruneOpts === false) return `url: ${pageUrl}\n` + raw;
+      const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '';
+      if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + raw;
       const pruned = pruneTree(result.tree, { mode: pruneOpts?.mode || 'act' });
       const out = formatTree(pruned);
       const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${out.length.toLocaleString()} chars (${Math.round((1 - out.length / raw.length) * 100)}% pruned)`;
-      return stats + '\n' + out;
+      return stats + '\n' + warn + out;
     },
 
     async click(ref) {
@@ -334,6 +341,8 @@ export async function connect(opts = {}) {
       writeFileSync(filePath, JSON.stringify(state, null, 2));
     },
 
+    get botBlocked() { return botBlocked; },
+
     dialogLog,
 
     async screenshot(screenshotOpts = {}) {
@@ -368,13 +377,17 @@ export async function connect(opts = {}) {
     async createTab() {
       const tab = await createPage(cdp, mode !== 'headed', { viewport: opts.viewport });
       await suppressPermissions(cdp);
+      let tabBotBlocked = false;
       return {
         async goto(url, timeout = 30000) {
           await navigate(tab, url, timeout);
           if (opts.consent !== false) {
             await dismissConsent(tab.session);
           }
+          const { tree, nodeCount } = await ariaTree(tab);
+          tabBotBlocked = isChallengePage(tree, nodeCount);
         },
+        get botBlocked() { return tabBotBlocked; },
         async injectCookies(url, cookieOpts) {
           await authenticate(tab.session, url, { browser: cookieOpts?.browser });
         },
@@ -485,7 +498,7 @@ async function ariaTree(page) {
     }
   }
 
-  return { tree, refMap };
+  return { tree, refMap, nodeCount: nodes.length };
 }
 
 /**
@@ -583,10 +596,14 @@ function waitForNetworkIdle(session, opts = {}) {
 
 /**
  * Detect if a page is a bot-challenge page (Cloudflare, etc.).
- * Heuristic: very short ARIA tree + known challenge phrases.
+ * Heuristic: low ARIA node count, short text, or known challenge phrases.
+ * @param {object} tree - Nested ARIA tree (from buildTree)
+ * @param {number} [nodeCount] - Raw CDP node count (from Accessibility.getFullAXTree)
  */
-function isChallengePage(tree) {
+function isChallengePage(tree, nodeCount) {
   if (!tree) return true;
+  // Real pages have 50+ ARIA nodes. Bot challenges have <20.
+  if (nodeCount !== undefined && nodeCount < 50) return true;
   const text = flattenTreeText(tree);
   // Near-empty pages are almost certainly blocks
   if (text.trim().length < 50) return true;