barebrowse 0.7.0 → 0.9.0

package/src/network-idle.js

@@ -0,0 +1,62 @@
+/**
+ * network-idle.js — wait until the page's network has been idle for N ms.
+ *
+ * Tracks in-flight requests by requestId in a Set, so an orphan
+ * loadingFinished/Failed (event for a request whose requestWillBeSent
+ * arrived before our listener attached) is a harmless no-op instead of
+ * driving a counter negative and resolving prematurely.
+ */
+
+/**
+ * @param {object} session - CDP session-scoped handle with .on() returning unsub
+ * @param {object} [opts]
+ * @param {number} [opts.timeout=30000] - Max wait time before reject
+ * @param {number} [opts.idle=500] - Required idle duration before resolve
+ */
+export function waitForNetworkIdle(session, opts = {}) {
+  const timeout = opts.timeout || 30000;
+  const idle = opts.idle || 500;
+
+  return new Promise((resolve, reject) => {
+    const pending = new Set();
+    let timer = null;
+    const unsubs = [];
+
+    const done = () => {
+      clearTimeout(timer);
+      clearTimeout(deadlineTimer);
+      for (const unsub of unsubs) unsub();
+      resolve();
+    };
+
+    const check = () => {
+      clearTimeout(timer);
+      if (pending.size === 0) {
+        timer = setTimeout(done, idle);
+      }
+    };
+
+    unsubs.push(session.on('Network.requestWillBeSent', (p) => {
+      pending.add(p.requestId);
+      clearTimeout(timer);
+    }));
+    unsubs.push(session.on('Network.loadingFinished', (p) => {
+      // delete() on a Set is a no-op for unknown keys — orphan events from
+      // requests started before we attached the listener can't push us negative.
+      pending.delete(p.requestId);
+      check();
+    }));
+    unsubs.push(session.on('Network.loadingFailed', (p) => {
+      pending.delete(p.requestId);
+      check();
+    }));
+
+    const deadlineTimer = setTimeout(() => {
+      for (const unsub of unsubs) unsub();
+      reject(new Error(`waitForNetworkIdle timed out after ${timeout}ms`));
+    }, timeout);
+
+    // Start check immediately (might already be idle)
+    check();
+  });
+}

package/src/stealth.js

@@ -23,28 +23,109 @@ const STEALTH_SCRIPT = `
     get: () => ['en-US', 'en'],
   });
 
-  // Fake chrome object (missing in headless)
-  if (!window.chrome) {
-    window.chrome = { runtime: {} };
+  // Realistic CPU + memory. Headless under containers can report 1 or odd
+  // values that real desktops rarely have, which is its own fingerprint.
+  Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
+  Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+
+  // chrome / chrome.runtime — headless either omits the object entirely or
+  // gives an empty {}; real Chrome has the enum shapes below even before any
+  // extension is installed. Fingerprinters check that chrome.runtime exists
+  // AND that these enums are present.
+  if (!window.chrome) window.chrome = {};
+  if (!window.chrome.runtime) {
+    window.chrome.runtime = {
+      OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
+      OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
+      PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+      PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
+      PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
+      RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' },
+    };
+  }
+
+  // Notification — headless Chrome doesn't expose the Notification API at
+  // all (even on secure contexts), while real Chrome always does and reports
+  // 'default' before any prompt. Fingerprinters check both \`typeof
+  // Notification\` and \`Notification.permission\`, so we fake both: the
+  // constructor when missing, and only the permission getter when it's
+  // present (some Chrome versions ship a non-configurable getter and
+  // defineProperty would throw — swallowed so the rest of the script runs).
+  if (typeof Notification === 'undefined') {
+    window.Notification = function Notification() {};
+    window.Notification.permission = 'default';
+    window.Notification.requestPermission = () => Promise.resolve('default');
+  } else {
+    try {
+      Object.defineProperty(Notification, 'permission', { get: () => 'default' });
+    } catch {}
   }
 
-  // Permissions.query: notifications return 'prompt' instead of 'denied'
+  // Permissions.query for notifications: keep it consistent with the
+  // Notification.permission override above instead of returning 'prompt'
+  // unconditionally (the prior hardcoded value was a tell of its own).
   const origQuery = Permissions.prototype.query;
   Permissions.prototype.query = function(desc) {
-    if (desc.name === 'notifications') {
-      return Promise.resolve({ state: 'prompt', onchange: null });
+    if (desc && desc.name === 'notifications') {
+      return Promise.resolve({ state: Notification.permission, onchange: null });
     }
     return origQuery.call(this, desc);
   };
+
+  // WebGL UNMASKED_VENDOR_WEBGL (37445) and UNMASKED_RENDERER_WEBGL (37446) —
+  // headless returns "Google Inc. (Google)" / "Google SwiftShader" which
+  // is the single most-used headless fingerprint. Spoof a realistic
+  // Intel integrated GPU pair (works on macOS and Linux user agents).
+  const SPOOFED_VENDOR = 'Intel Inc.';
+  const SPOOFED_RENDERER = 'Intel Iris OpenGL Engine';
+  const origGetParam = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(p) {
+    if (p === 37445) return SPOOFED_VENDOR;
+    if (p === 37446) return SPOOFED_RENDERER;
+    return origGetParam.apply(this, arguments);
+  };
+  if (typeof WebGL2RenderingContext !== 'undefined') {
+    const origGetParam2 = WebGL2RenderingContext.prototype.getParameter;
+    WebGL2RenderingContext.prototype.getParameter = function(p) {
+      if (p === 37445) return SPOOFED_VENDOR;
+      if (p === 37446) return SPOOFED_RENDERER;
+      return origGetParam2.apply(this, arguments);
+    };
+  }
 `;
 
 /**
  * Apply stealth patches to a CDP session.
  * Must be called before any navigation.
  *
+ * Splits into two layers:
+ *   1. Network.setUserAgentOverride strips "HeadlessChrome" from the UA
+ *      that ships in HTTP request headers AND that navigator.userAgent
+ *      reports — `--headless=new` leaves "HeadlessChrome" in there.
+ *   2. Page.addScriptToEvaluateOnNewDocument injects the JS-level patches
+ *      before any page script runs.
+ *
  * @param {object} session - Session-scoped CDP handle
  */
 export async function applyStealth(session) {
+  // 1. UA override — read whatever the running browser actually claims, then
+  //    rewrite the "Headless" marker out. Doing it this way (vs hardcoding a
+  //    string) keeps the version + platform fields accurate across Chromium
+  //    releases. Network.setUserAgentOverride is per-session, so it also
+  //    cleans up the value navigator.userAgent reports inside the page.
+  try {
+    const { userAgent } = await session.send('Browser.getVersion');
+    if (userAgent && userAgent.includes('HeadlessChrome')) {
+      await session.send('Network.setUserAgentOverride', {
+        userAgent: userAgent.replace(/HeadlessChrome/g, 'Chrome'),
+      });
+    }
+  } catch {
+    // Browser.getVersion not reachable from this session — skip UA override
+    // and rely on the JS-level patches alone.
+  }
+
+  // 2. JS-level patches
   await session.send('Page.addScriptToEvaluateOnNewDocument', {
     source: STEALTH_SCRIPT,
   });

package/.aurora/plans/active/harden-assess/design.md

@@ -1,68 +0,0 @@
-# Design: Harden assess tool
-
-## Architecture
-
-### Current flow (broken)
-```
-assess(url) → connect({ mode: 'hybrid' })  ← NEW browser, no cookies
-            → assessFn(page, url)
-            → page.close()                  ← kills browser
-```
-
-### New flow
-```
-assess(url) → getPage()                     ← reuse session browser
-            → page.createTab()              ← new tab in same browser
-            → tab.injectCookies(url)        ← cookie injection
-            → assessFn(tab, url)            ← assess uses tab
-            → tab.close()                   ← close tab only
-            timeout guard wraps entire flow
-            retry wraps entire flow
-```
-
-## Key design decisions
-
-### Why a new tab, not the session page?
-wearehere's `assess()` calls:
-- `session.send('Page.addScriptToEvaluateOnNewDocument', ...)` — injects fingerprint detection scripts
-- `networkSession.on('Network.requestWillBeSent', ...)` — monitors all network traffic
-
-These would pollute the session page. A separate tab has its own CDP session with isolated Page/Network domains.
-
-### createTab() page-like interface
-wearehere expects a page object with: `goto()`, `cdp` (raw session), `waitForNetworkIdle()`. createTab() returns exactly this interface:
-
-```javascript
-{
-  goto(url, timeout)        // navigate tab
-  cdp                       // raw CDP session for this tab
-  waitForNetworkIdle(opts)  // reuses existing waitForNetworkIdle()
-  injectCookies(url)        // cookie injection for this tab
-  close()                   // close tab, NOT the browser
-}
-```
-
-### Retry strategy
-```
-attempt 1: assess with 45s timeout
-  fail → wait 2s
-attempt 2: assess with 45s timeout (if browser crashed, reset _page first)
-  fail → return { error: "assessment_failed", ... }
-```
-
-### Timeout implementation
-```javascript
-const result = await Promise.race([
-  doAssess(page, url, opts),
-  new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 45000))
-]);
-```
-On timeout, the tab is closed in a finally block.
-
-## Files changed
-| File | Change |
-|------|--------|
-| `src/index.js` | Add `createTab()` and tab's `close()` to connect() return object |
-| `mcp-server.js` | Rewrite assess case: use getPage().createTab(), add retry + timeout |
-| `README.md` | Update assess description, add self-healing mention |
-| `barebrowse.context.md` | Update assess section, document createTab() |

package/.aurora/plans/active/harden-assess/plan.md

@@ -1,71 +0,0 @@
-# Plan: Harden assess tool — session reuse + self-healing
-
-## Plan ID
-`harden-assess`
-
-## Summary
-Make the assess tool reuse the MCP session's browser instance (cookies, headed fallback) instead of spawning throwaway browsers, add retry logic for transient failures, and add a timeout guard so no single assessment can hang forever.
-
-## Problem Statement
-The `assess` tool in `mcp-server.js` calls `connect({ mode: 'hybrid' })` directly — creating a fresh headless browser per call with no cookies, no session state, and no headed fallback. Every other MCP tool uses the `getPage()` singleton. This causes:
-
-1. **No cookies** — assess browses as a stranger, getting blocked by consent walls and bot detection that cookies would bypass
-2. **No headed fallback reuse** — if the singleton already fell back to headed mode, assess still starts fresh headless and hits the same blocks
-3. **No retry** — any failure (browser crash, navigation timeout, CDP disconnect) kills the assessment with no recovery
-4. **No timeout guard** — if `wearehere`'s `assess()` hangs (e.g. network idle never resolves), the MCP call blocks indefinitely
-
-However, assess can't simply use the shared `_page` singleton directly because `wearehere` injects init scripts (`addScriptToEvaluateOnNewDocument`) and network listeners that would pollute the session page for subsequent calls. The solution is to create a **second page tab** within the same browser instance.
-
-## Proposed Solution
-1. **Session-aware page creation** — Instead of `connect()`, assess opens a new CDP tab within the existing browser. This shares the browser process (same cookies, same headed/headless state) but isolates assess's script injections to its own tab.
-2. **Retry with backoff** — Wrap the assess call in a retry loop (max 2 attempts, 2s backoff). On browser crash, reconnect the singleton.
-3. **Timeout guard** — Wrap each assess call in `Promise.race` with a 45s hard deadline. If exceeded, return an error result (not hang).
-
-## Benefits
-- Assess gets cookies and headed fallback for free — no separate browser instance
-- Failed assessments auto-retry instead of dying
-- Hanging assessments time out gracefully instead of blocking the MCP server forever
-- Eliminates the 10+ second cold-start per assessment (browser launch)
-
-## Scope
-### In Scope
-- Modify `mcp-server.js` assess handler to create tabs within existing browser
-- Add `createTab()` / `closeTab()` helper to `src/index.js` connect() page handle
-- Add retry wrapper in mcp-server.js
-- Add timeout guard in mcp-server.js
-- Update docs: README.md, barebrowse.context.md, CLAUDE.md
-
-### Out of Scope
-- Changing wearehere's internal logic
-- Adding retry/self-healing to other MCP tools (future work)
-- Batch/queue mode for multiple assessments
-- Changing the assess tool's MCP interface (same inputs/outputs)
-
-## Dependencies
-- `wearehere` package (assess function signature unchanged)
-- `src/index.js` connect() API (adding createTab/closeTab methods)
-- `src/cdp.js` (Target.createTarget / closeTarget already available)
-
-## Implementation Strategy
-Phase 1: Add tab management to connect() page handle (createTab, closeTab)
-Phase 2: Rewrite assess handler to use session tab + retry + timeout
-Phase 3: Update documentation
-
-## Risks and Mitigations
-| Risk | Impact | Mitigation |
-|------|--------|------------|
-| Init script injection leaks across tabs | Pollutes session page | Each tab gets its own Page domain; addScriptToEvaluateOnNewDocument is per-target |
-| Browser crash during assess kills session too | Session page lost | getPage() already handles reconnection lazily (set _page = null, next call recreates) |
-| wearehere expects full page handle, not raw CDP session | API mismatch | createTab() returns a page-like object with goto, cdp, waitForNetworkIdle — same interface |
-
-## Success Criteria
-- [ ] `assess` reuses the session browser (no separate `connect()` call)
-- [ ] `assess` inherits cookies from the session
-- [ ] `assess` works when session is in headed mode (hybrid fallback already triggered)
-- [ ] Failed assessment retries once before returning error
-- [ ] Assessment hanging > 45s returns timeout error, doesn't block server
-- [ ] All existing tests pass
-- [ ] Documentation updated
-
-## Open Questions
-1. Should createTab() also inject cookies? — **Recommendation**: Yes, call `authenticate()` for the target URL before navigation, same as `goto` does.

package/.aurora/plans/active/harden-assess/prd.md

@@ -1,38 +0,0 @@
-# PRD: Harden assess tool
-
-## Overview
-The `assess` MCP tool must reuse the session browser, retry on failure, and time out gracefully.
-
-## Requirements
-
-### R1: Session-aware tab creation
-The assess tool MUST create a new browser tab within the existing MCP session browser instead of spawning a separate browser via `connect()`. The tab MUST:
-- Share the same browser process (inheriting headless/headed state)
-- Have access to the browser's cookie jar
-- Isolate its CDP domains (Page, Network, DOM) from the session page
-- Be closed after each assessment completes or fails
-
-### R2: Cookie injection
-Before navigating, the assess tab MUST inject cookies from the user's browser for the target URL, using the same `authenticate()` mechanism as `goto`.
-
-### R3: Retry on failure
-If an assessment fails (navigation timeout, CDP error, browser crash), the tool MUST:
-- Retry once after a 2-second delay
-- If the browser crashed, reset the session singleton (`_page = null`) so getPage() reconnects
-- If retry also fails, return a structured error result (not throw)
-
-### R4: Timeout guard
-Each assessment MUST have a hard timeout of 45 seconds. If exceeded:
-- The tab is force-closed
-- A structured error result is returned: `{ site, url, error: "timeout", scanned_at }`
-- The session page is NOT affected
-
-### R5: Backwards compatibility
-- The assess tool's MCP interface (inputs/outputs) MUST NOT change
-- Successful assessments return the same JSON format as before
-- The tool appears/disappears based on wearehere availability (unchanged)
-
-## Non-functional
-- No new dependencies
-- No changes to wearehere package
-- createTab()/closeTab() exposed on connect() page handle for library users too