neoagent 3.0.1-beta.27 → 3.0.1-beta.29

package/docs/automation.md

@@ -62,6 +62,23 @@ when no reply is needed.
 Prefer official integrations or MCP tools over browser automation when both
 can perform the action. Structured tools are easier to restrict and diagnose.
 
+## Bot-protected pages
+
+The isolated VM browser uses native anti-detection hardening inspired by
+Botasaurus' public anti-detect browser patterns: stable browser profiles,
+common desktop viewport sizes, less mechanical pointer movement, referrer-aware
+navigation, and challenge detection telemetry. NeoAgent does not embed or
+depend on Botasaurus for this path.
+
+`browser_navigate` accepts optional `referrerMode` values of `direct`,
+`google`, or `current`. By default it also retries once with a Google referrer
+when a known bot challenge is detected. Browser results can include
+`botDetection` so runs can report that a page is blocked instead of treating the
+challenge page as ordinary content.
+
+Browser automation still must comply with the target site's terms and applicable
+law. URL safety checks and the VM device-access restrictions remain in effect.
+
 ## Runs and delivery
 
 Open **Runs** to inspect the trigger, tool calls, approvals, output, and error

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neoagent",
-  "version": "3.0.1-beta.27",
+  "version": "3.0.1-beta.29",
   "description": "Self-hosted AI agent for long-running tasks, automation, messaging, device control, and local memory",
   "license": "AGPL-3.0-only",
   "main": "server/index.js",

package/server/public/.last_build_id

@@ -1 +1 @@
-802b144119e4d7e88f089829dde8e27b
+8b097147c5ec8ac3bee082f22bd5d248

package/server/public/flutter_bootstrap.js

@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"a10d8ac38de835021c8d2f920dbf50a920ccc0
 
 _flutter.loader.load({
   serviceWorkerSettings: {
-    serviceWorkerVersion: "3226159482" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
+    serviceWorkerVersion: "1039759445" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
   }
 });

package/server/public/main.dart.js

@@ -138719,7 +138719,7 @@ r===$&&A.b()
 p.push(A.iU(q,A.ju(!1,new A.Y(B.vr,A.cN(new A.cA(B.jU,new A.a94(r,q),q),q,q),q),!1,B.I,!0),q,q,0,0,0,q))}r=!1
 if(!s.ay)if(!s.ch){r=s.e
 r===$&&A.b()
-r=B.b.t("mqv0w1e9-4502c91").length!==0&&r.b}if(r){r=s.d
+r=B.b.t("mr0924tg-6e8948b").length!==0&&r.b}if(r){r=s.d
 r===$&&A.b()
 r=r.aD&&!r.av?84:0
 s=s.e
@@ -145015,7 +145015,7 @@ $S:0}
 A.a0g.prototype={}
 A.U0.prototype={
 nm(a){var s=this
-if(B.b.t("mqv0w1e9-4502c91").length===0||s.a!=null)return
+if(B.b.t("mr0924tg-6e8948b").length===0||s.a!=null)return
 s.Bg()
 s.a=A.m4(B.Td,new A.bgn(s))},
 Bg(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f
@@ -145033,7 +145033,7 @@ if(!t.f.b(k)){s=1
 break}i=J.a2(k,"buildId")
 h=i==null?null:B.b.t(J.p(i))
 j=h==null?"":h
-if(J.bd(j)===0||J.f(j,"mqv0w1e9-4502c91")){s=1
+if(J.bd(j)===0||J.f(j,"mr0924tg-6e8948b")){s=1
 break}n.b=!0
 n.E()
 p=2
@@ -145050,7 +145050,7 @@ case 2:return A.i(o.at(-1),r)}})
 return A.k($async$Bg,r)},
 vV(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f,e,d,c,b,a,a0,a1
 var $async$vV=A.h(function(a2,a3){if(a2===1){o.push(a3)
-s=p}for(;;)switch(s){case 0:if(B.b.t("mqv0w1e9-4502c91").length===0||n.c){s=1
+s=p}for(;;)switch(s){case 0:if(B.b.t("mr0924tg-6e8948b").length===0||n.c){s=1
 break}n.c=!0
 n.E()
 p=4

package/server/routes/browser.js

@@ -103,7 +103,14 @@ router.post('/navigate', async (req, res) => {
       return res.status(403).json({ error: 'This URL is not permitted.' });
     }
 
-    const result = await bc.navigate(url, { waitUntil: waitFor || 'domcontentloaded' });
+    const result = await bc.navigate(url, {
+      waitUntil: waitFor || req.body?.waitUntil || 'domcontentloaded',
+      waitFor,
+      screenshot: req.body?.screenshot !== false,
+      fullPage: req.body?.fullPage === true,
+      referrerMode: req.body?.referrerMode,
+      challengeRetry: req.body?.challengeRetry,
+    });
     res.json(result);
   } catch (err) {
     res.status(500).json({ error: sanitizeError(err) });

package/server/services/ai/rate_limits.js

@@ -9,6 +9,13 @@ const _reservations = new Map();
 const DEFAULT_RATE_LIMIT_4H = 2_500_000;
 const DEFAULT_RATE_LIMIT_WEEKLY = 10_000_000;
 
+// Generous upper bound on a single run's token cost. Used to size the in-flight
+// reservation that keeps concurrent run starts from collectively bypassing the
+// per-user budget. It must NOT be the full limit — doing so would reserve the
+// entire budget for one run and reject any second concurrent run for the same
+// user (e.g. a scheduled task firing in the same minute as another).
+const DEFAULT_RUN_RESERVATION_TOKENS = 500_000;
+
 const WINDOWS = {
   fourHour: {
     durationMs: 4 * 60 * 60 * 1000,
@@ -144,10 +151,22 @@ function enforceRateLimits(userId) {
     throw new RateLimitExceededError('weekly', snapshot);
   }
   // Reserve a placeholder so concurrent starts see this run as in-flight.
-  // The reservation is released (or reconciled) when the run completes.
+  // The reservation is released (or reconciled) when the run completes. Size it
+  // to a realistic per-run estimate — never the full limit — so concurrent runs
+  // for the same user are admitted normally and only a genuinely near-budget
+  // user gets throttled. Clamp so the reservation can never by itself exceed a
+  // (possibly small/custom) limit.
   const key = String(userId);
   const limits = snapshot.limits;
-  const reserve = Math.max(limits.fourHour || 0, limits.weekly || 0, 1);
+  const estimate = parsePositiveInteger(
+    process.env.NEOAGENT_RUN_RESERVATION_TOKENS,
+    DEFAULT_RUN_RESERVATION_TOKENS,
+  );
+  const cap = Math.min(
+    limits.fourHour == null ? Infinity : limits.fourHour,
+    limits.weekly == null ? Infinity : limits.weekly,
+  );
+  const reserve = Math.max(1, Math.min(estimate, Number.isFinite(cap) ? cap : estimate));
   _reservations.set(key, (_reservations.get(key) || 0) + reserve);
   return { snapshot, releaseReservation: () => releaseReservation(userId, reserve) };
 }

package/server/services/ai/tools.js

@@ -437,7 +437,9 @@ function getAvailableTools(app, options = {}) {
                     url: { type: 'string', description: 'URL to navigate to' },
                     screenshot: { type: 'boolean', description: 'Take a screenshot (default true)' },
                     waitFor: { type: 'string', description: 'CSS selector to wait for' },
-                    fullPage: { type: 'boolean', description: 'Full page screenshot (default false)' }
+                    fullPage: { type: 'boolean', description: 'Full page screenshot (default false)' },
+                    referrerMode: { type: 'string', enum: ['direct', 'google', 'current'], description: 'Navigation referrer strategy for the VM browser (default direct). google sends a Google referrer; current navigates from the current page when possible.' },
+                    challengeRetry: { type: 'boolean', description: 'Retry once with a Google referrer if a known bot challenge is detected (default true).' }
                 },
                 required: ['url']
             }
@@ -1787,7 +1789,9 @@ async function executeTool(toolName, args, context, engine) {
             return { ...await provider.navigate(args.url, {
                 screenshot: args.screenshot !== false,
                 waitFor: args.waitFor,
-                fullPage: args.fullPage
+                fullPage: args.fullPage,
+                referrerMode: args.referrerMode,
+                challengeRetry: args.challengeRetry
             }), backend };
         }
 

package/server/services/browser/anti_detection.js

@@ -0,0 +1,192 @@
+'use strict';
+
+const crypto = require('crypto');
+
+const USER_AGENTS = Object.freeze([
+  {
+    ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    platform: 'Win32',
+    webglVendor: 'Google Inc. (NVIDIA)',
+    webglRenderer: 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)',
+  },
+  {
+    ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
+    platform: 'Win32',
+    webglVendor: 'Google Inc. (Intel)',
+    webglRenderer: 'ANGLE (Intel, Intel(R) UHD Graphics 770 Direct3D11 vs_5_0 ps_5_0, D3D11)',
+  },
+  {
+    ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    platform: 'MacIntel',
+    webglVendor: 'Google Inc. (Apple)',
+    webglRenderer: 'ANGLE (Apple, Apple M2, OpenGL 4.1)',
+  },
+  {
+    ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
+    platform: 'MacIntel',
+    webglVendor: 'Google Inc. (Apple)',
+    webglRenderer: 'ANGLE (Apple, Apple M1, OpenGL 4.1)',
+  },
+  {
+    ua: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    platform: 'Linux x86_64',
+    webglVendor: 'Google Inc. (Intel)',
+    webglRenderer: 'ANGLE (Intel, Mesa Intel(R) UHD Graphics 620, OpenGL 4.6)',
+  },
+]);
+
+const VIEWPORTS = Object.freeze([
+  { width: 1920, height: 1080, weight: 35 },
+  { width: 1366, height: 768, weight: 26 },
+  { width: 1536, height: 864, weight: 16 },
+  { width: 1280, height: 720, weight: 9 },
+  { width: 1440, height: 900, weight: 9 },
+  { width: 1600, height: 900, weight: 5 },
+]);
+
+const REFERRER_MODES = new Set(['direct', 'google', 'current']);
+
+function hashInt(value) {
+  const hash = crypto.createHash('sha256').update(String(value || 'default')).digest();
+  return hash.readUInt32BE(0);
+}
+
+function chooseWeighted(items, seed) {
+  const total = items.reduce((sum, item) => sum + Number(item.weight || 1), 0);
+  let cursor = Math.abs(Number(seed) || 0) % total;
+  for (const item of items) {
+    cursor -= Number(item.weight || 1);
+    if (cursor < 0) return item;
+  }
+  return items[0];
+}
+
+function chooseBrowserIdentity(profileKey) {
+  const seed = hashInt(profileKey);
+  const profile = USER_AGENTS[seed % USER_AGENTS.length];
+  const viewport = chooseWeighted(VIEWPORTS, seed >>> 3);
+  return {
+    userAgent: profile.ua,
+    platform: profile.platform,
+    viewport: { width: viewport.width, height: viewport.height },
+    webglVendor: profile.webglVendor,
+    webglRenderer: profile.webglRenderer,
+    hardwareConcurrency: [4, 6, 8, 12, 16][seed % 5],
+    deviceMemory: [4, 8, 16][(seed >>> 5) % 3],
+  };
+}
+
+function normalizeReferrerMode(value) {
+  const normalized = String(value || '').trim().toLowerCase();
+  return REFERRER_MODES.has(normalized) ? normalized : 'direct';
+}
+
+function normalizeChallengeRetry(value) {
+  return value !== false;
+}
+
+function detectBotChallenge(input = {}) {
+  const title = String(input.title || '').trim();
+  const url = String(input.url || '').trim();
+  const html = String(input.html || '');
+  const text = String(input.text || input.pageContent || '');
+  const haystack = `${title}\n${url}\n${html.slice(0, 200000)}\n${text.slice(0, 50000)}`.toLowerCase();
+
+  if (title === 'Just a moment...' || haystack.includes('cf-turnstile-response') || haystack.includes('/cdn-cgi/challenge-platform/')) {
+    return { detected: true, provider: 'cloudflare' };
+  }
+  if (haystack.includes('please verify you are a human') || haystack.includes('press & hold to confirm')) {
+    return { detected: true, provider: 'perimeterx' };
+  }
+  if (haystack.includes('datadome') || haystack.includes('geo.captcha-delivery.com')) {
+    return { detected: true, provider: 'datadome' };
+  }
+  if (
+    haystack.includes('unusual traffic')
+    || haystack.includes('are you a robot')
+    || haystack.includes('verify that you are human')
+    || haystack.includes('complete the security check')
+  ) {
+    return { detected: true, provider: 'generic' };
+  }
+  return { detected: false, provider: null };
+}
+
+function rand(min, max) {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function easeInOutQuad(t) {
+  return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
+}
+
+function easeOutQuad(t) {
+  return 1 - (1 - t) * (1 - t);
+}
+
+function easeInOutSine(t) {
+  return -(Math.cos(Math.PI * t) - 1) / 2;
+}
+
+function bezierPoint(points, t) {
+  let current = points.map((point) => ({ x: point.x, y: point.y }));
+  while (current.length > 1) {
+    current = current.slice(0, -1).map((point, index) => ({
+      x: point.x + (current[index + 1].x - point.x) * t,
+      y: point.y + (current[index + 1].y - point.y) * t,
+    }));
+  }
+  return current[0];
+}
+
+function generateHumanMousePath(from, to, viewport = {}) {
+  const start = {
+    x: Number.isFinite(Number(from?.x)) ? Number(from.x) : 0,
+    y: Number.isFinite(Number(from?.y)) ? Number(from.y) : 0,
+  };
+  const end = {
+    x: Number.isFinite(Number(to?.x)) ? Number(to.x) : 0,
+    y: Number.isFinite(Number(to?.y)) ? Number(to.y) : 0,
+  };
+  const distance = Math.hypot(end.x - start.x, end.y - start.y);
+  if (distance < 2) return [start, end];
+
+  const width = Math.max(1, Number(viewport.width || 1440));
+  const height = Math.max(1, Number(viewport.height || 900));
+  const steps = Math.max(12, Math.min(90, Math.round(18 + distance / 18 + rand(-5, 8))));
+  const boundaryX = Math.min(90, Math.max(10, distance * 0.12));
+  const boundaryY = Math.min(90, Math.max(10, distance * 0.12));
+  const controlCount = rand(1, 3);
+  const controls = [];
+  for (let i = 1; i <= controlCount; i += 1) {
+    const ratio = i / (controlCount + 1);
+    controls.push({
+      x: Math.max(0, Math.min(width, start.x + (end.x - start.x) * ratio + rand(-boundaryX, boundaryX))),
+      y: Math.max(0, Math.min(height, start.y + (end.y - start.y) * ratio + rand(-boundaryY, boundaryY))),
+    });
+  }
+
+  const tweens = [easeInOutQuad, easeOutQuad, easeInOutSine];
+  const tween = tweens[rand(0, tweens.length - 1)];
+  const curve = [start, ...controls, end];
+  const path = [];
+  for (let i = 0; i < steps; i += 1) {
+    const point = bezierPoint(curve, tween(i / (steps - 1)));
+    path.push({
+      x: Math.round(Math.max(0, Math.min(width, point.x))),
+      y: Math.round(Math.max(0, Math.min(height, point.y + (i > 0 && i < steps - 1 ? rand(-1, 1) : 0)))),
+    });
+  }
+  path[0] = { x: Math.round(start.x), y: Math.round(start.y) };
+  path[path.length - 1] = { x: Math.round(end.x), y: Math.round(end.y) };
+  return path;
+}
+
+module.exports = {
+  chooseBrowserIdentity,
+  detectBotChallenge,
+  generateHumanMousePath,
+  normalizeChallengeRetry,
+  normalizeReferrerMode,
+  rand,
+};

package/server/services/browser/controller.js

@@ -2,6 +2,14 @@ const path = require('path');
 const fs = require('fs');
 const { spawn } = require('child_process');
 const { DATA_DIR } = require('../../../runtime/paths');
+const {
+  chooseBrowserIdentity,
+  detectBotChallenge,
+  generateHumanMousePath,
+  normalizeChallengeRetry,
+  normalizeReferrerMode,
+  rand,
+} = require('./anti_detection');
 
 const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
 if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
@@ -9,21 +17,6 @@ const BROWSER_PROFILE_ROOT = path.join(DATA_DIR, 'browser-profiles');
 if (!fs.existsSync(BROWSER_PROFILE_ROOT)) fs.mkdirSync(BROWSER_PROFILE_ROOT, { recursive: true });
 const BROWSER_READY_MARKER = '/var/lib/neoagent/browser-runtime-ready';
 
-const USER_AGENTS = [
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
-];
-
-const VIEWPORTS = [
-  { width: 1280, height: 800 },
-  { width: 1366, height: 768 },
-  { width: 1440, height: 900 },
-  { width: 1920, height: 1080 },
-];
-
 // Injected into every page in the cloud VM browser context to deny local device access.
 const DEVICE_DENY_SCRIPT = `(() => {
   const denied = () => Promise.reject(Object.assign(
@@ -166,10 +159,6 @@ function installPlaywrightBrowserBinary(browserName) {
   });
 }
 
-function rand(min, max) {
-  return Math.floor(Math.random() * (max - min + 1)) + min;
-}
-
 function sleep(ms) {
   return new Promise(r => setTimeout(r, ms));
 }
@@ -241,10 +230,15 @@ class BrowserController {
     this.launchPromise = null;
     this.browserBinaryInstallPromise = null;
     this.headless = false;
-    this._viewport = VIEWPORTS[0];
-    this._userAgent = USER_AGENTS[0];
     this.profileDir = path.join(BROWSER_PROFILE_ROOT, this.userId || 'default');
     if (!fs.existsSync(this.profileDir)) fs.mkdirSync(this.profileDir, { recursive: true });
+    this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
+    this._viewport = this._identity.viewport;
+    this._userAgent = this._identity.userAgent;
+    this._mousePosition = {
+      x: Math.round(this._viewport.width / 2),
+      y: Math.round(this._viewport.height / 2),
+    };
   }
 
   async setHeadless(val) {
@@ -260,6 +254,7 @@ class BrowserController {
   async _applyStealthToPage(page) {
     const ua = this._userAgent;
     const vp = this._viewport;
+    const identity = this._identity || chooseBrowserIdentity(this.userId || this.profileDir);
 
     if (typeof page.setUserAgent === 'function') {
       await page.setUserAgent(ua);
@@ -281,9 +276,9 @@ class BrowserController {
 
         // Realistic language/platform
         Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-        Object.defineProperty(navigator, 'platform', { get: () => 'MacIntel' });
-        Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${rand(4, 16)} });
-        Object.defineProperty(navigator, 'deviceMemory', { get: () => ${[4, 8, 16][rand(0, 2)]} });
+        Object.defineProperty(navigator, 'platform', { get: () => ${JSON.stringify(identity.platform)} });
+        Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => ${Number(identity.hardwareConcurrency) || 8} });
+        Object.defineProperty(navigator, 'deviceMemory', { get: () => ${Number(identity.deviceMemory) || 8} });
 
         // Make it look like a real Chrome install
         window.chrome = {
@@ -323,9 +318,9 @@ class BrowserController {
           apply: function(target, ctx, args) {
             const param = args[0];
             // UNMASKED_VENDOR_WEBGL
-            if (param === 37445) return 'Google Inc. (Apple)';
+            if (param === 37445) return ${JSON.stringify(identity.webglVendor)};
             // UNMASKED_RENDERER_WEBGL
-            if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
+            if (param === 37446) return ${JSON.stringify(identity.webglRenderer)};
             return Reflect.apply(target, ctx, args);
           }
         };
@@ -388,8 +383,13 @@ class BrowserController {
       }
       await this.ensureVirtualDisplay();
 
-      this._userAgent = USER_AGENTS[rand(0, USER_AGENTS.length - 1)];
-      this._viewport = VIEWPORTS[rand(0, VIEWPORTS.length - 1)];
+      this._identity = chooseBrowserIdentity(this.userId || this.profileDir);
+      this._userAgent = this._identity.userAgent;
+      this._viewport = this._identity.viewport;
+      this._mousePosition = {
+        x: Math.round(this._viewport.width / 2),
+        y: Math.round(this._viewport.height / 2),
+      };
 
       let executablePath = resolveBrowserExecutablePath();
       if (!executablePath) {
@@ -414,18 +414,28 @@ class BrowserController {
       };
 
       const launchArgs = [
+        '--start-maximized',
+        '--remote-allow-origins=*',
         '--no-sandbox',
         '--disable-setuid-sandbox',
         '--disable-dev-shm-usage',
+        '--no-service-autorun',
         '--disable-crash-reporter',
+        '--disable-breakpad',
         '--disable-background-networking',
         '--disable-component-update',
         '--disable-blink-features=AutomationControlled',
         '--disable-infobars',
+        '--disable-session-crashed-bubble',
+        '--disable-search-engine-choice-screen',
         '--no-first-run',
         '--no-default-browser-check',
+        '--homepage=about:blank',
+        '--no-pings',
+        '--password-store=basic',
         '--disable-gpu',
         '--lang=en-US,en',
+        `--user-agent=${this._userAgent}`,
         `--window-size=${this._viewport.width},${this._viewport.height}`,
         // Cloud security: disable hardware device APIs at the Chromium level
         '--disable-features=WebBluetooth,WebUSB,WebSerial,WebOTP,DirectSockets',
@@ -440,6 +450,8 @@ class BrowserController {
         env: launchEnv,
         args: launchArgs,
         viewport: this._viewport,
+        userAgent: this._userAgent,
+        locale: 'en-US',
         ignoreHTTPSErrors: false,
         timeout: 120000,
       });
@@ -535,13 +547,48 @@ class BrowserController {
     return page.screenshot(screenshotOptions);
   }
 
+  async _navigatePage(page, url, options = {}) {
+    const referrerMode = normalizeReferrerMode(options.referrerMode);
+    const waitUntil = normalizeWaitUntil(options.waitUntil);
+    if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
+      const previousUrl = page.url();
+      await page.evaluate((nextUrl) => { window.location.href = nextUrl; }, url);
+      await page.waitForFunction((oldUrl) => window.location.href !== oldUrl, previousUrl, { timeout: 10000 }).catch(() => {});
+      await page.waitForLoadState(waitUntil, { timeout: 30000 }).catch(() => {});
+      return null;
+    }
+
+    const gotoOptions = {
+      waitUntil,
+      timeout: 30000,
+    };
+    if (referrerMode === 'google') {
+      gotoOptions.referer = 'https://www.google.com/';
+    } else if (referrerMode === 'current' && page.url() && page.url() !== 'about:blank') {
+      gotoOptions.referer = page.url();
+    }
+    return page.goto(url, gotoOptions);
+  }
+
+  async _getBotDetection(page, rawHtml, pageContent) {
+    const title = await page.title().catch(() => '');
+    return detectBotChallenge({
+      title,
+      url: page.url(),
+      html: rawHtml,
+      pageContent,
+    });
+  }
+
   async navigate(url, options = {}) {
     const page = await this.ensurePage();
 
     try {
-      const response = await page.goto(url, {
-        waitUntil: normalizeWaitUntil(options.waitUntil),
-        timeout: 30000
+      const requestedReferrerMode = normalizeReferrerMode(options.referrerMode);
+      let activeReferrerMode = requestedReferrerMode;
+      let response = await this._navigatePage(page, url, {
+        ...options,
+        referrerMode: activeReferrerMode,
       });
 
       if (options.waitFor) {
@@ -549,7 +596,34 @@ class BrowserController {
       }
 
       // Simulate human reading delay
-      await sleep(rand(500, 1500));
+      await sleep(rand(700, 1800));
+
+      let rawHtml = await page.content();
+      const { extractForLLM } = require('./contentExtractor');
+      let extraction = extractForLLM(rawHtml, { url: page.url() });
+      let botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
+      let challengeRetried = false;
+
+      if (
+        botDetection.detected
+        && normalizeChallengeRetry(options.challengeRetry)
+        && requestedReferrerMode === 'direct'
+      ) {
+        challengeRetried = true;
+        activeReferrerMode = 'google';
+        await sleep(rand(1200, 2600));
+        response = await this._navigatePage(page, url, {
+          ...options,
+          referrerMode: activeReferrerMode,
+        });
+        if (options.waitFor) {
+          await page.waitForSelector(options.waitFor, { timeout: 10000 }).catch(() => { });
+        }
+        await sleep(rand(900, 2200));
+        rawHtml = await page.content();
+        extraction = extractForLLM(rawHtml, { url: page.url() });
+        botDetection = await this._getBotDetection(page, rawHtml, extraction.markdown);
+      }
 
       const title = await page.title();
       const currentUrl = page.url();
@@ -559,15 +633,14 @@ class BrowserController {
         screenshot = await this.takeScreenshot({ fullPage: options.fullPage });
       }
 
-      const rawHtml = await page.content();
-      const { extractForLLM } = require('./contentExtractor');
-      const extraction = extractForLLM(rawHtml, { url: currentUrl });
-
       return {
         title,
         url: currentUrl,
         status: response?.status() || 0,
         pageContent: extraction.markdown,
+        botDetection,
+        referrerMode: activeReferrerMode,
+        challengeRetried,
         screenshotPath: screenshot?.screenshotPath || null,
         artifactId: screenshot?.artifactId || null,
         fullPath: screenshot?.fullPath || null
@@ -578,6 +651,7 @@ class BrowserController {
       return {
         error: err.message,
         url,
+        botDetection: { detected: false, provider: null },
         screenshotPath: screenshot?.screenshotPath || null,
         artifactId: screenshot?.artifactId || null,
         fullPath: screenshot?.fullPath || null
@@ -585,6 +659,36 @@ class BrowserController {
     }
   }
 
+  async _moveMouseTo(page, x, y, options = {}) {
+    const target = {
+      x: Math.max(0, Math.min(this._viewport.width, Math.round(Number(x) || 0))),
+      y: Math.max(0, Math.min(this._viewport.height, Math.round(Number(y) || 0))),
+    };
+    const path = generateHumanMousePath(this._mousePosition, target, this._viewport);
+    for (const point of path) {
+      await page.mouse.move(point.x, point.y);
+      if (!options.fast) {
+        await sleep(rand(2, 10));
+      }
+    }
+    this._mousePosition = target;
+    return target;
+  }
+
+  async _pointForElement(element) {
+    await element.scrollIntoViewIfNeeded?.().catch(() => {});
+    const box = await element.boundingBox();
+    if (!box || box.width <= 0 || box.height <= 0) {
+      return null;
+    }
+    const xRatio = rand(25, 75) / 100;
+    const yRatio = rand(30, 70) / 100;
+    return {
+      x: box.x + box.width * xRatio,
+      y: box.y + box.height * yRatio,
+    };
+  }
+
   async click(selector, text, screenshot = true) {
     const page = await this.ensurePage();
 
@@ -608,10 +712,14 @@ class BrowserController {
         return { error: 'Either selector or text required' };
       }
 
-      // Human-like: hover first, then click with a hold delay
-      await target.hover();
-      await sleep(rand(80, 250));
-      await target.click({ delay: rand(50, 150) });
+      const point = await this._pointForElement(target);
+      if (!point) return { error: 'Element has no visible clickable area' };
+
+      await this._moveMouseTo(page, point.x, point.y);
+      await sleep(rand(170, 320));
+      await page.mouse.down();
+      await sleep(rand(80, 260));
+      await page.mouse.up();
 
       await sleep(rand(800, 1800));
 
@@ -637,10 +745,10 @@ class BrowserController {
     try {
       const px = Math.max(0, Math.round(Number(x) || 0));
       const py = Math.max(0, Math.round(Number(y) || 0));
-      await page.mouse.move(px, py, { steps: rand(4, 10) });
-      await sleep(rand(40, 140));
+      await this._moveMouseTo(page, px, py);
+      await sleep(rand(90, 220));
       await page.mouse.down();
-      await sleep(rand(30, 110));
+      await sleep(rand(70, 240));
       await page.mouse.up();
       await sleep(rand(500, 1200));
 
@@ -667,7 +775,7 @@ class BrowserController {
     try {
       const px = Math.max(0, Math.round(Number(x) || 0));
       const py = Math.max(0, Math.round(Number(y) || 0));
-      await page.mouse.move(px, py, { steps: options.steps || 1 });
+      await this._moveMouseTo(page, px, py, { fast: Number(options.steps) <= 1 });
       return {
         success: true,
         x: px,
@@ -684,11 +792,19 @@ class BrowserController {
     const page = await this.ensurePage();
 
     try {
-      await page.mouse.wheel({
-        deltaX: Math.round(Number(deltaX) || 0),
-        deltaY: Math.round(Number(deltaY) || 0),
-      });
-      await sleep(rand(300, 900));
+      const x = Math.max(10, Math.min(this._viewport.width - 10, this._mousePosition.x + rand(-80, 80)));
+      const y = Math.max(10, Math.min(this._viewport.height - 10, this._mousePosition.y + rand(-80, 80)));
+      await this._moveMouseTo(page, x, y);
+      const totalX = Math.round(Number(deltaX) || 0);
+      const totalY = Math.round(Number(deltaY) || 0);
+      const chunks = Math.max(1, Math.min(6, Math.ceil(Math.max(Math.abs(totalX), Math.abs(totalY)) / 450)));
+      for (let i = 0; i < chunks; i += 1) {
+        await page.mouse.wheel({
+          deltaX: Math.round(totalX / chunks),
+          deltaY: Math.round(totalY / chunks),
+        });
+        await sleep(rand(120, 360));
+      }
 
       let screenshotResult = null;
       if (screenshot) screenshotResult = await this.takeScreenshot();
@@ -711,12 +827,20 @@ class BrowserController {
 
     try {
       if (options.clear !== false) {
-        await page.click(selector, { clickCount: 3 });
+        const element = await page.$(selector);
+        if (element) {
+          const point = await this._pointForElement(element);
+          if (point) {
+            await this._moveMouseTo(page, point.x, point.y);
+            await sleep(rand(120, 260));
+          }
+        }
+        await page.click(selector, { clickCount: 3, delay: rand(40, 120) });
         await page.keyboard.press('Backspace');
       }
 
       for (const char of text) {
-        await page.type(selector, char, { delay: rand(30, 150) });
+        await page.type(selector, char, { delay: rand(45, 180) });
       }
 
       if (options.pressEnter) {
@@ -744,7 +868,7 @@ class BrowserController {
 
     try {
       for (const char of String(text || '')) {
-        await page.keyboard.type(char, { delay: rand(25, 110) });
+        await page.keyboard.type(char, { delay: rand(45, 160) });
       }
 
       if (options.pressEnter) {
@@ -797,6 +921,8 @@ class BrowserController {
     const page = await this.ensurePage();
 
     try {
+      const rawHtml = await page.content().catch(() => '');
+      const botDetection = await this._getBotDetection(page, rawHtml, '').catch(() => ({ detected: false, provider: null }));
       if (all) {
         const results = await page.$$eval(selector || 'body', (elements, attr) => {
           return elements.map(el => {
@@ -806,7 +932,7 @@ class BrowserController {
             return el.innerText || '';
           });
         }, attribute);
-        return { results: results.slice(0, 100) };
+        return { results: results.slice(0, 100), botDetection };
       }
 
       const result = await page.$eval(selector || 'body', (el, attr) => {
@@ -816,7 +942,7 @@ class BrowserController {
         return el.innerText || '';
       }, attribute);
 
-      return { result: typeof result === 'string' ? result.slice(0, 50000) : result };
+      return { result: typeof result === 'string' ? result.slice(0, 50000) : result, botDetection };
     } catch (err) {
       return { error: err.message };
     }