browserclaw 0.6.0 → 0.7.0

package/dist/index.cjs

@@ -1295,7 +1295,20 @@ async function fetchChromeVersion(cdpUrl, timeoutMs = 500, authToken) {
 async function isChromeReachable(cdpUrl, timeoutMs = 500, authToken) {
   if (isWebSocketUrl(cdpUrl)) return await canOpenWebSocket(cdpUrl, timeoutMs);
   const version = await fetchChromeVersion(cdpUrl, timeoutMs, authToken);
-  return Boolean(version);
+  if (version !== null) return true;
+  let isLoopback = false;
+  try {
+    const u = new URL(cdpUrl.startsWith("http") ? cdpUrl : `http://${cdpUrl}`);
+    isLoopback = isLoopbackHost(u.hostname);
+  } catch {
+  }
+  if (!isLoopback) return false;
+  for (let i = 0; i < 2; i++) {
+    await new Promise((r) => setTimeout(r, 150));
+    const retry = await fetchChromeVersion(cdpUrl, timeoutMs, authToken);
+    if (retry !== null) return true;
+  }
+  return false;
 }
 async function getChromeWebSocketUrl(cdpUrl, timeoutMs = 500, authToken) {
   if (isWebSocketUrl(cdpUrl)) return cdpUrl;
@@ -1487,6 +1500,198 @@ async function stopChrome(running, timeoutMs = 2500) {
   }
 }
 
+// src/stealth.ts
+var STEALTH_SCRIPT = `(function() {
+  'use strict';
+  function p(fn) { try { fn(); } catch(_) {} }
+
+  // \u2500\u2500 1. navigator.webdriver \u2192 undefined \u2500\u2500
+  p(function() {
+    Object.defineProperty(navigator, 'webdriver', { get: function() { return undefined; }, configurable: true });
+  });
+
+  // \u2500\u2500 2. navigator.plugins + mimeTypes (only if empty \u2014 Chrome 92+ populates them natively) \u2500\u2500
+  p(function() {
+    if (navigator.plugins && navigator.plugins.length > 0) return;
+
+    function FakePlugin(name, fn, desc, mimes) {
+      this.name = name; this.filename = fn; this.description = desc; this.length = mimes.length;
+      for (var i = 0; i < mimes.length; i++) { this[i] = mimes[i]; mimes[i].enabledPlugin = this; }
+    }
+    FakePlugin.prototype.item = function(i) { return this[i] || null; };
+    FakePlugin.prototype.namedItem = function(n) {
+      for (var i = 0; i < this.length; i++) if (this[i].type === n) return this[i];
+      return null;
+    };
+
+    function M(type, suf, desc) { this.type = type; this.suffixes = suf; this.description = desc; }
+
+    var m1 = new M('application/pdf', 'pdf', 'Portable Document Format');
+    var m2 = new M('application/x-google-chrome-pdf', 'pdf', 'Portable Document Format');
+    var m3 = new M('application/x-nacl', '', 'Native Client Executable');
+    var m4 = new M('application/x-pnacl', '', 'Portable Native Client Executable');
+
+    var plugins = [
+      new FakePlugin('Chrome PDF Plugin', 'internal-pdf-viewer', 'Portable Document Format', [m1]),
+      new FakePlugin('Chrome PDF Viewer', 'mhjfbmdgcfjbbpaeojofohoefgiehjai', '', [m2]),
+      new FakePlugin('Native Client', 'internal-nacl-plugin', '', [m3, m4]),
+    ];
+
+    function makeIterable(arr, items) {
+      arr.length = items.length;
+      for (var i = 0; i < items.length; i++) arr[i] = items[i];
+      arr[Symbol.iterator] = function() {
+        var idx = 0;
+        return { next: function() {
+          return idx < items.length ? { value: items[idx++], done: false } : { done: true };
+        }};
+      };
+    }
+
+    var pa = { item: function(i) { return plugins[i] || null; },
+      namedItem: function(n) { for (var i = 0; i < plugins.length; i++) if (plugins[i].name === n) return plugins[i]; return null; },
+      refresh: function() {} };
+    makeIterable(pa, plugins);
+    Object.defineProperty(navigator, 'plugins', { get: function() { return pa; } });
+
+    var allMimes = [m1, m2, m3, m4];
+    var ma = { item: function(i) { return allMimes[i] || null; },
+      namedItem: function(n) { for (var i = 0; i < allMimes.length; i++) if (allMimes[i].type === n) return allMimes[i]; return null; } };
+    makeIterable(ma, allMimes);
+    Object.defineProperty(navigator, 'mimeTypes', { get: function() { return ma; } });
+  });
+
+  // \u2500\u2500 3. navigator.languages (cached + frozen so identity check passes) \u2500\u2500
+  p(function() {
+    if (!navigator.languages || navigator.languages.length === 0) {
+      var langs = Object.freeze(['en-US', 'en']);
+      Object.defineProperty(navigator, 'languages', { get: function() { return langs; } });
+    }
+  });
+
+  // \u2500\u2500 4. window.chrome \u2500\u2500
+  p(function() {
+    if (window.chrome && window.chrome.runtime && window.chrome.runtime.connect) return;
+
+    var chrome = window.chrome || {};
+    var noop = function() {};
+    var evtStub = { addListener: noop, removeListener: noop, hasListeners: function() { return false; } };
+    chrome.runtime = chrome.runtime || {};
+    chrome.runtime.onMessage = chrome.runtime.onMessage || evtStub;
+    chrome.runtime.onConnect = chrome.runtime.onConnect || evtStub;
+    chrome.runtime.sendMessage = chrome.runtime.sendMessage || noop;
+    chrome.runtime.connect = chrome.runtime.connect || function() {
+      return { onMessage: { addListener: noop }, postMessage: noop, disconnect: noop };
+    };
+    if (chrome.runtime.id === undefined) chrome.runtime.id = undefined;
+    if (!chrome.loadTimes) chrome.loadTimes = function() { return {}; };
+    if (!chrome.csi) chrome.csi = function() { return {}; };
+    if (!chrome.app) {
+      chrome.app = {
+        isInstalled: false,
+        InstallState: { INSTALLED: 'installed', NOT_INSTALLED: 'not_installed', DISABLED: 'disabled' },
+        RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' },
+        getDetails: function() { return null; },
+        getIsInstalled: function() { return false; },
+        runningState: function() { return 'cannot_run'; },
+      };
+    }
+
+    if (!window.chrome) {
+      Object.defineProperty(window, 'chrome', { value: chrome, writable: false, enumerable: true, configurable: false });
+    }
+  });
+
+  // \u2500\u2500 5. Permissions API consistency \u2500\u2500
+  p(function() {
+    var orig = navigator.permissions.query.bind(navigator.permissions);
+    function q(params) {
+      if (params.name === 'notifications') {
+        return Promise.resolve({
+          state: typeof Notification !== 'undefined' ? Notification.permission : 'prompt',
+          name: 'notifications', onchange: null,
+          addEventListener: function(){}, removeEventListener: function(){}, dispatchEvent: function(){ return true; },
+        });
+      }
+      return orig(params);
+    }
+    q.toString = function() { return 'function query() { [native code] }'; };
+    navigator.permissions.query = q;
+  });
+
+  // \u2500\u2500 6. WebGL vendor / renderer \u2500\u2500
+  p(function() {
+    var h = {
+      apply: function(target, self, args) {
+        var param = args[0];
+        if (param === 0x9245) return 'Intel Inc.';
+        if (param === 0x9246) return 'Intel Iris OpenGL Engine';
+        return Reflect.apply(target, self, args);
+      }
+    };
+    if (typeof WebGLRenderingContext !== 'undefined')
+      WebGLRenderingContext.prototype.getParameter = new Proxy(WebGLRenderingContext.prototype.getParameter, h);
+    if (typeof WebGL2RenderingContext !== 'undefined')
+      WebGL2RenderingContext.prototype.getParameter = new Proxy(WebGL2RenderingContext.prototype.getParameter, h);
+  });
+
+  // \u2500\u2500 7. Notification.permission \u2500\u2500
+  p(function() {
+    if (typeof Notification !== 'undefined' && Notification.permission === 'denied') {
+      Object.defineProperty(Notification, 'permission', { get: function() { return 'default'; }, configurable: true });
+    }
+  });
+
+  // \u2500\u2500 8. navigator.connection (cached so identity check passes) \u2500\u2500
+  p(function() {
+    if (navigator.connection) return;
+    var conn = {
+      effectiveType: '4g', rtt: 50, downlink: 10, saveData: false, onchange: null,
+      addEventListener: function(){}, removeEventListener: function(){}, dispatchEvent: function(){ return true; },
+    };
+    Object.defineProperty(navigator, 'connection', { get: function() { return conn; } });
+  });
+
+  // \u2500\u2500 9. Iframe contentWindow.chrome \u2500\u2500
+  // Handled by patch 4 \u2014 chrome object is now on window, propagates to iframes on same origin.
+
+  // \u2500\u2500 10. console method toString \u2500\u2500
+  p(function() {
+    ['log','info','warn','error','debug','table','trace'].forEach(function(n) {
+      if (console[n]) {
+        console[n].toString = function() { return 'function ' + n + '() { [native code] }'; };
+      }
+    });
+  });
+
+  // \u2500\u2500 11. Headless-mode window / screen fixes \u2500\u2500
+  p(function() {
+    if (window.outerWidth === 0)
+      Object.defineProperty(window, 'outerWidth', { get: function() { return window.innerWidth || 1920; } });
+    if (window.outerHeight === 0)
+      Object.defineProperty(window, 'outerHeight', { get: function() { return (window.innerHeight || 1080) + 85; } });
+  });
+
+  p(function() {
+    if (screen.colorDepth === 0) {
+      Object.defineProperty(screen, 'colorDepth', { get: function() { return 24; } });
+      Object.defineProperty(screen, 'pixelDepth', { get: function() { return 24; } });
+    }
+  });
+
+  // \u2500\u2500 12. navigator.hardwareConcurrency \u2500\u2500
+  p(function() {
+    if (!navigator.hardwareConcurrency)
+      Object.defineProperty(navigator, 'hardwareConcurrency', { get: function() { return 4; } });
+  });
+
+  // \u2500\u2500 13. navigator.deviceMemory \u2500\u2500
+  p(function() {
+    if (!navigator.deviceMemory)
+      Object.defineProperty(navigator, 'deviceMemory', { get: function() { return 8; } });
+  });
+})()`;
+
 // src/connection.ts
 var BrowserTabNotFoundError = class extends Error {
   constructor(message = "Tab not found") {
@@ -1727,7 +1932,6 @@ function ensurePageState(page) {
   }
   return state;
 }
-var STEALTH_SCRIPT = `Object.defineProperty(navigator, 'webdriver', { get: () => undefined })`;
 function applyStealthToPage(page) {
   page.evaluate(STEALTH_SCRIPT).catch((e) => {
     if (process.env.DEBUG !== void 0 && process.env.DEBUG !== "")
@@ -3547,6 +3751,99 @@ async function setTimezoneViaPlaywright(opts) {
   });
 }
 
+// src/anti-bot.ts
+var DETECT_CHALLENGE_SCRIPT = `(function() {
+  var title = (document.title || '').toLowerCase();
+
+  // Cloudflare JS challenge
+  if (title === 'just a moment...'
+      || document.querySelector('#challenge-running, #cf-please-wait, #challenge-form')
+      || title.indexOf('checking your browser') !== -1) {
+    return { kind: 'cloudflare-js', message: 'Cloudflare JS challenge' };
+  }
+
+  // Cloudflare block page (needs body text \u2014 read lazily)
+  var body = null;
+  function getBody() { if (body === null) body = (document.body && document.body.textContent) || ''; return body; }
+
+  if (title.indexOf('attention required') !== -1
+      || (document.querySelector('.cf-error-details') && getBody().indexOf('blocked') !== -1)) {
+    return { kind: 'cloudflare-block', message: 'Cloudflare block page' };
+  }
+
+  // Cloudflare Turnstile
+  if (document.querySelector('.cf-turnstile, iframe[src*="challenges.cloudflare.com"]')) {
+    return { kind: 'cloudflare-turnstile', message: 'Cloudflare Turnstile challenge' };
+  }
+
+  // hCaptcha
+  if (document.querySelector('.h-captcha, iframe[src*="hcaptcha.com"]')) {
+    return { kind: 'hcaptcha', message: 'hCaptcha challenge' };
+  }
+
+  // reCAPTCHA
+  if (document.querySelector('.g-recaptcha, iframe[src*="google.com/recaptcha"]')) {
+    return { kind: 'recaptcha', message: 'reCAPTCHA challenge' };
+  }
+
+  // Generic access-denied / rate-limit pages (only read body for short pages)
+  var b = getBody();
+  if (b.length < 5000) {
+    if (/access denied|403 forbidden/i.test(title) || /access denied/i.test(b)) {
+      return { kind: 'blocked', message: 'Access denied' };
+    }
+    if (/\\b429\\b/i.test(title) || /too many requests|rate limit/i.test(b)) {
+      return { kind: 'rate-limited', message: 'Rate limited' };
+    }
+  }
+
+  return null;
+})()`;
+function parseChallengeResult(raw) {
+  if (raw !== null && typeof raw === "object" && "kind" in raw) {
+    return raw;
+  }
+  return null;
+}
+async function detectChallengeViaPlaywright(opts) {
+  const page = await getPageForTargetId({ cdpUrl: opts.cdpUrl, targetId: opts.targetId });
+  ensurePageState(page);
+  return parseChallengeResult(await page.evaluate(DETECT_CHALLENGE_SCRIPT));
+}
+async function waitForChallengeViaPlaywright(opts) {
+  const page = await getPageForTargetId({ cdpUrl: opts.cdpUrl, targetId: opts.targetId });
+  ensurePageState(page);
+  const timeout = normalizeTimeoutMs(opts.timeoutMs, 15e3);
+  const poll = Math.max(250, Math.min(5e3, opts.pollMs ?? 500));
+  const detect = async () => parseChallengeResult(await page.evaluate(DETECT_CHALLENGE_SCRIPT));
+  const initial = await detect();
+  if (initial === null) return { resolved: true, challenge: null };
+  if (initial.kind === "cloudflare-js") {
+    try {
+      await page.waitForFunction(
+        "document.title.toLowerCase() !== 'just a moment...' && !document.querySelector('#challenge-running')",
+        void 0,
+        { timeout }
+      );
+      await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => {
+      });
+      const after = await detect();
+      return { resolved: after === null, challenge: after };
+    } catch {
+      const after = await detect();
+      return { resolved: after === null, challenge: after };
+    }
+  }
+  const deadline = Date.now() + timeout;
+  while (Date.now() < deadline) {
+    await page.waitForTimeout(poll);
+    const current = await detect();
+    if (current === null) return { resolved: true, challenge: null };
+  }
+  const final = await detect();
+  return { resolved: final === null, challenge: final };
+}
+
 // src/capture/activity.ts
 function consolePriority(level) {
   switch (level) {
@@ -5151,6 +5448,53 @@ var CrawlPage = class {
       name
     });
   }
+  // ── Anti-Bot ──────────────────────────────────────────────────
+  /**
+   * Detect whether the page is showing an anti-bot challenge
+   * (Cloudflare, hCaptcha, reCAPTCHA, access-denied, rate-limit, etc.).
+   *
+   * Returns `null` if no challenge is detected.
+   *
+   * @example
+   * ```ts
+   * const challenge = await page.detectChallenge();
+   * if (challenge) {
+   *   console.log(challenge.kind);    // 'cloudflare-js'
+   *   console.log(challenge.message); // 'Cloudflare JS challenge'
+   * }
+   * ```
+   */
+  async detectChallenge() {
+    return detectChallengeViaPlaywright({ cdpUrl: this.cdpUrl, targetId: this.targetId });
+  }
+  /**
+   * Wait for an anti-bot challenge to resolve on its own.
+   *
+   * Cloudflare JS challenges typically auto-resolve in ~5 seconds.
+   * CAPTCHA challenges will only resolve if solved in a visible browser window.
+   *
+   * @param opts.timeoutMs - Maximum wait time (default: `15000`)
+   * @param opts.pollMs - Poll interval (default: `500`)
+   * @returns Whether the challenge resolved, and the remaining challenge info if not
+   *
+   * @example
+   * ```ts
+   * await page.goto('https://example.com');
+   * const challenge = await page.detectChallenge();
+   * if (challenge?.kind === 'cloudflare-js') {
+   *   const { resolved } = await page.waitForChallenge({ timeoutMs: 20000 });
+   *   if (!resolved) throw new Error('Challenge did not resolve');
+   * }
+   * ```
+   */
+  async waitForChallenge(opts) {
+    return waitForChallengeViaPlaywright({
+      cdpUrl: this.cdpUrl,
+      targetId: this.targetId,
+      timeoutMs: opts?.timeoutMs,
+      pollMs: opts?.pollMs
+    });
+  }
 };
 var BrowserClaw = class _BrowserClaw {
   cdpUrl;
@@ -5300,12 +5644,14 @@ exports.BrowserClaw = BrowserClaw;
 exports.BrowserTabNotFoundError = BrowserTabNotFoundError;
 exports.CrawlPage = CrawlPage;
 exports.InvalidBrowserNavigationUrlError = InvalidBrowserNavigationUrlError;
+exports.STEALTH_SCRIPT = STEALTH_SCRIPT;
 exports.assertBrowserNavigationAllowed = assertBrowserNavigationAllowed;
 exports.assertBrowserNavigationRedirectChainAllowed = assertBrowserNavigationRedirectChainAllowed;
 exports.assertBrowserNavigationResultAllowed = assertBrowserNavigationResultAllowed;
 exports.assertSafeUploadPaths = assertSafeUploadPaths;
 exports.batchViaPlaywright = batchViaPlaywright;
 exports.createPinnedLookup = createPinnedLookup;
+exports.detectChallengeViaPlaywright = detectChallengeViaPlaywright;
 exports.ensureContextState = ensureContextState;
 exports.executeSingleAction = executeSingleAction;
 exports.forceDisconnectPlaywrightForTarget = forceDisconnectPlaywrightForTarget;
@@ -5324,6 +5670,7 @@ exports.resolvePageByTargetIdOrThrow = resolvePageByTargetIdOrThrow;
 exports.resolvePinnedHostnameWithPolicy = resolvePinnedHostnameWithPolicy;
 exports.resolveStrictExistingUploadPaths = resolveStrictExistingUploadPaths;
 exports.sanitizeUntrustedFileName = sanitizeUntrustedFileName;
+exports.waitForChallengeViaPlaywright = waitForChallengeViaPlaywright;
 exports.withBrowserNavigationPolicy = withBrowserNavigationPolicy;
 exports.withPageScopedCdpClient = withPageScopedCdpClient;
 exports.withPlaywrightPageCdpSession = withPlaywrightPageCdpSession;