human-browser 4.3.4 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "human-browser",
-  "version": "4.3.4",
+  "version": "4.4.0",
   "description": "Stealth browser for AI agents. Bypasses Cloudflare, DataDome, PerimeterX. Residential IPs from 10+ countries. iPhone 15 Pro fingerprint. Drop-in Playwright replacement — launchHuman() just works.",
   "keywords": [
     "browser-automation",

package/scripts/browser-human.js

@@ -25,20 +25,60 @@
 // ─── PLAYWRIGHT RESOLVER ──────────────────────────────────────────────────────
 // Works in any context: clawhub install, workspace, Clawster containers
 
+// Try patchright first (drop-in Playwright fork with CDP-leak patches:
+// no Runtime.enable, no HeadlessChrome UA, navigator.webdriver removed at
+// source level). Fall back to vanilla playwright if patchright is not
+// installed — keeps the npm package usable without the optional dep.
 function _requirePlaywright() {
+  const fs = require('fs');
   const tries = [
-    () => require('playwright'),
-    () => require(`${__dirname}/../node_modules/playwright`),
-    () => require(`${__dirname}/../../node_modules/playwright`),
-    () => require(`${process.env.HOME || '/root'}/.openclaw/workspace/node_modules/playwright`),
-    () => require('./node_modules/playwright'),
+    ['patchright',                                                            () => require('patchright')],
+    ['patchright (server/node_modules)',                                      () => require(`${__dirname}/../node_modules/patchright`)],
+    ['patchright (workspace)',                                                () => require(`${__dirname}/../../node_modules/patchright`)],
+    ['patchright (server/prototype/node_modules)',                            () => require(`${__dirname}/../prototype/node_modules/patchright`)],
+    ['patchright (/app/prototype)',                                           () => require('/app/prototype/node_modules/patchright')],
+    ['playwright',                                                            () => require('playwright')],
+    ['playwright (server/node_modules)',                                      () => require(`${__dirname}/../node_modules/playwright`)],
+    ['playwright (workspace)',                                                () => require(`${__dirname}/../../node_modules/playwright`)],
+    ['playwright (HOME workspace)',                                           () => require(`${process.env.HOME || '/root'}/.openclaw/workspace/node_modules/playwright`)],
+    ['playwright (./node_modules)',                                           () => require('./node_modules/playwright')],
   ];
-  for (const fn of tries) {
-    try { return fn(); } catch (_) {}
+  // Verify the chromium executable is actually present before accepting a
+  // module — patchright pins to a different chromium revision than playwright,
+  // and v48 production was broken because patchright was loadable but its
+  // chromium-1217 binary wasn't installed in the image.
+  function _executableExists(mod) {
+    try {
+      const exe = mod.chromium.executablePath();
+      return exe && fs.existsSync(exe);
+    } catch (_) {
+      return false;
+    }
+  }
+  let lastValid = null;
+  let lastValidLabel = null;
+  for (const [label, fn] of tries) {
+    try {
+      const mod = fn();
+      if (_executableExists(mod)) {
+        try { console.log(`[human-browser] launcher using ${label}`); } catch (_) {}
+        return mod;
+      }
+      // Module loadable but executable missing — keep looking for a usable backend.
+      if (!lastValid) { lastValid = mod; lastValidLabel = label; }
+      try { console.warn(`[human-browser] ${label} loaded but chromium binary missing — trying next`); } catch (_) {}
+    } catch (_) {}
+  }
+  // No backend has its chromium installed. Return the first loadable module
+  // anyway; the eventual launch() will throw a clear "Executable doesn't exist"
+  // message that surfaces in session-server logs.
+  if (lastValid) {
+    try { console.warn(`[human-browser] falling back to ${lastValidLabel} (chromium binary missing — launch will fail)`); } catch (_) {}
+    return lastValid;
   }
   throw new Error(
-    '[human-browser] playwright not found.\n' +
-    'Run: npm install playwright && npx playwright install chromium'
+    '[human-browser] neither patchright nor playwright found.\n' +
+    'Run: npm install patchright playwright && npx playwright install chromium && npx patchright install chromium'
   );
 }
 
@@ -111,9 +151,16 @@ function buildDevice(mobile, country = 'ro') {
 
 const PROXY_PRESETS = {
   decodo: {
-    // Sticky session via port number: each unique port = unique sticky IP
-    serverTemplate: (country, port) => `http://${country}.decodo.com:${port}`,
-    usernameTemplate: (user) => user,
+    // Canonical Decodo entrypoint: gate.decodo.com:7000 (rotating endpoint)
+    // with sticky session + country encoded in username (`user-` prefix +
+    // `-country-XX-session-Y-sessionduration-30`). Verified: country IS
+    // enforced this way; the port-range form (gate.decodo.com:10001..49999)
+    // pins sticky IP but IGNORES country in username — that's why us-zone was
+    // leaking UK Sky Broadband IPs. The country-subdomain form
+    // (us.decodo.com:port) is the same legacy soft-routing path.
+    serverTemplate: (country, port) => `http://gate.decodo.com:7000`,
+    usernameTemplate: (user, country, port) =>
+      `user-${user}-country-${country}-session-${port}-sessionduration-30`,
     defaultUser: null,
     defaultPass: null,
     defaultCountry: 'ro',
@@ -490,9 +537,35 @@ async function launchHuman(opts = {}) {
     '--disable-setuid-sandbox',
     '--ignore-certificate-errors',
     '--disable-blink-features=AutomationControlled',
-    '--disable-features=IsolateOrigins,site-per-process',
+    // QUIC over UDP can't traverse an HTTP CONNECT proxy; without this
+    // Chromium spends 30s+ retrying QUIC against google.com before TCP
+    // fallback, blowing the bubus NavigateToUrlEvent budget.
+    '--disable-quic',
+    // Kill startup chatter that races Playwright's CDP-based proxy auth
+    // interceptor (Chromium asks for Proxy-Authorization only after a 407;
+    // dozens of concurrent startup fetches all hit 407 simultaneously and
+    // some land on a tunnel the proxy already dropped → "duplicate response"
+    // CDP warnings + 60s+ NavigateToUrlEvent timeouts).
+    '--disable-features=IsolateOrigins,site-per-process,UseDnsHttpsSvcb,UseDnsHttpsSvcbAlpn,AsyncDns,OptimizationHints,OptimizationGuideModelDownloading,OptimizationTargetPrediction,InterestFeedContentSuggestions,Translate,MediaRouter',
+    '--disable-background-networking',
+    '--disable-component-update',
+    '--disable-sync',
+    '--disable-domain-reliability',
+    '--no-default-browser-check',
+    '--no-first-run',
+    '--no-pings',
+    // Belt-and-braces: tell Chrome to never bypass the proxy locally.
+    '--proxy-bypass-list=<-loopback>',
   ];
-  if (cdpPort) launchArgs.push(`--remote-debugging-port=${cdpPort}`);
+  if (cdpPort) {
+    launchArgs.push(`--remote-debugging-port=${cdpPort}`);
+    // Chrome 111+ already blocks page-origin DevTools WS by default, but make
+    // the policy explicit and future-proof: only allow connections from
+    // server-side WS clients (no Origin header) — anything claiming a real
+    // page origin is rejected. Setting an unreachable HTTPS sentinel keeps
+    // the spec-required allowlist non-empty without granting any real origin.
+    launchArgs.push('--remote-allow-origins=https://disabled.invalid');
+  }
 
   const effectiveHeadless = headed ? false : headless;
 
@@ -550,6 +623,84 @@ async function launchHuman(opts = {}) {
     }
   }, { mobile, locale: meta.locale });
 
+  // Bot-friendly URL rewrites: top-level navigations only, applied via
+  // ctx.route(). Reddit's www/new front-end is heavily Cloudflare-bot-blocked
+  // for fresh residential IPs; old.reddit.com is on the same auth/cookie space
+  // but ships a much lighter (and far less protected) HTML page. Net effect:
+  // higher pass-through rate without changing any user-visible profile state.
+  await ctx.route(/^https?:\/\/(www\.|new\.|m\.)?reddit\.com\//i, (route) => {
+    try {
+      const orig = route.request().url();
+      // Only rewrite navigation/document loads — leave XHR/static asset paths
+      // alone so login flows and OAuth don't break.
+      if (route.request().resourceType() !== 'document') return route.continue();
+      const rewritten = orig.replace(
+        /^(https?:\/\/)(?:www\.|new\.|m\.)?reddit\.com\//i,
+        '$1old.reddit.com/'
+      );
+      if (rewritten === orig) return route.continue();
+      console.warn(`[human-browser] rewrite reddit ${orig} -> ${rewritten}`);
+      return route.continue({ url: rewritten });
+    } catch (_) { try { route.continue(); } catch (_) {} }
+  });
+
+  // Anti-bot response logger. Catches main-frame responses with status that
+  // indicates a block/challenge (403/429/451/503) or known CF/Akamai/PerimeterX
+  // signatures, and emits a single-line log so the operator can see at a glance
+  // when sites are pushing back. Body sniffing is best-effort and bounded
+  // (first 4KB) to avoid memory issues on huge pages.
+  ctx.on('response', async (resp) => {
+    try {
+      const req = resp.request();
+      // Only main document loads — not images/fonts/scripts.
+      if (req.resourceType() !== 'document') return;
+      const url = req.url();
+      const status = resp.status();
+      let host = '';
+      try { host = new URL(url).host; } catch (_) {}
+
+      // Status-based ban signals.
+      const banStatus = status === 403 || status === 429 || status === 451 ||
+                         (status === 503 && host !== '127.0.0.1');
+
+      // Header signatures (Cloudflare's challenge / hCaptcha / Akamai BMP).
+      let banReason = null;
+      const hdrs = resp.headers();
+      const cfChlg = hdrs['cf-mitigated'] || hdrs['cf-chl-bypass'] || '';
+      const server = (hdrs['server'] || '').toLowerCase();
+      const xrh = (hdrs['x-robots-tag'] || '').toLowerCase();
+      if (cfChlg) banReason = `cf-mitigated:${cfChlg}`;
+      else if (server.includes('akamaighost') && status >= 400) banReason = 'akamai-block';
+      else if (banStatus) banReason = `status-${status}`;
+
+      if (!banReason) return;
+
+      // Best-effort body sniff for inline reason. Kept tiny.
+      let bodyHint = '';
+      try {
+        const buf = await resp.body();
+        const txt = buf.slice(0, 4096).toString('utf8').replace(/\s+/g, ' ');
+        // Pull a few telltale phrases.
+        const matchers = [
+          /just a moment/i, /attention required/i, /access denied/i,
+          /blocked.{0,40}network security/i, /verifying you are human/i,
+          /your request has been blocked/i, /unusual traffic/i,
+          /captcha/i, /perimeterx/i, /datadome/i, /forbidden/i,
+        ];
+        for (const re of matchers) {
+          const m = txt.match(re);
+          if (m) { bodyHint = m[0].slice(0, 80); break; }
+        }
+      } catch (_) {}
+
+      console.warn(
+        `[human-browser] BLOCKED host=${host} status=${status} reason=${banReason}` +
+        (bodyHint ? ` hint="${bodyHint}"` : '') +
+        ` url=${url.slice(0, 200)}`
+      );
+    } catch (_) { /* listener must never throw */ }
+  });
+
   // Persistent context launches with a default page; reuse it instead of
   // opening a second tab (ephemeral context starts with no pages).
   const existing = ctx.pages();
@@ -573,6 +724,12 @@ async function launchHuman(opts = {}) {
   return {
     browser, ctx, page,
     cdpHttpUrl, cdpWsUrl,
+    proxy, // resolved {server, username, password} or null — needed so callers
+           // (session-server.js → browser-use-runner.py) can hand the same creds
+           // to browser-use's root-CDP proxy auth handler. Without this, browser-use
+           // creates its own targets via raw CDP and Chromium returns 407 because
+           // patchright's Fetch.authRequired interceptor is bound per-CRPage, not
+           // browser-wide.
     humanClick, humanMouseMove, humanType, humanScroll, humanRead, sleep, rand,
   };
 }