imperium-crawl 2.4.0 → 2.5.2

package/dist/stealth/browser-image-extract.js

@@ -0,0 +1,268 @@
+/**
+ * Browser-based image extraction engine for imperium-crawl v2.5.1
+ * Executes inside the browser via page.evaluate() for 100% coverage.
+ *
+ * Discovers images from:
+ * - <img> tags (src, data-src, data-lazy-src, srcset)
+ * - <picture> + <source srcset>
+ * - Inline style="background-image:url(...)"
+ * - All <style> tags → parsed CSS rules → background-image
+ * - Shadow DOM — recursively walks all web components
+ * - JSON-LD <script type="application/ld+json"> → image/images fields
+ * - Inline <script> window.__INITIAL_STATE__ / __DATA__ → regex image URLs
+ * - Same-origin iframes → recursive DOM scan
+ *
+ * Triggers lazy-load via scroll, optionally auto-clicks "load more" buttons.
+ */
+import { debugLog } from "../utils/debug.js";
+// ── Browser-side extraction script (runs inside page.evaluate) ──
+const EXTRACTION_SCRIPT = `
+(() => {
+  const results = [];
+  const seen = new Set();
+
+  function resolveUrl(href, base) {
+    if (!href || href.startsWith("data:")) return null;
+    try {
+      return new URL(href, base).href;
+    } catch { return null; }
+  }
+
+  function addImage(url, alt, width, height, selector, source) {
+    if (!url || seen.has(url)) return;
+    seen.add(url);
+    results.push({ url, alt: alt || "", width: width || 0, height: height || 0, selector, source });
+  }
+
+  function extractFromRoot(root, docUrl, prefix) {
+    // 1. <img> tags
+    root.querySelectorAll("img").forEach((img) => {
+      const src = img.currentSrc || img.src || img.dataset.src || img.dataset.lazySrc || img.getAttribute("data-src");
+      if (src) {
+        const url = resolveUrl(src, docUrl);
+        if (url) addImage(url, img.alt, img.naturalWidth, img.naturalHeight, prefix + getSelector(img), "img");
+      }
+      // srcset
+      const srcset = img.srcset || img.getAttribute("data-srcset");
+      if (srcset) {
+        srcset.split(",").forEach((s) => {
+          const u = s.trim().split(/\\s+/)[0];
+          const url = resolveUrl(u, docUrl);
+          if (url) addImage(url, img.alt, img.naturalWidth, img.naturalHeight, prefix + getSelector(img), "img");
+        });
+      }
+    });
+
+    // 2. <picture> + <source srcset>
+    root.querySelectorAll("picture").forEach((pic) => {
+      pic.querySelectorAll("source").forEach((source) => {
+        const srcset = source.srcset || source.getAttribute("data-srcset");
+        if (srcset) {
+          srcset.split(",").forEach((s) => {
+            const u = s.trim().split(/\\s+/)[0];
+            const url = resolveUrl(u, docUrl);
+            if (url) addImage(url, "", 0, 0, prefix + getSelector(pic), "picture");
+          });
+        }
+      });
+    });
+
+    // 3. Inline style background-image
+    root.querySelectorAll("*[style]").forEach((el) => {
+      const style = el.getAttribute("style") || "";
+      const match = style.match(/background-image\\s*:\\s*url\\([\"']?([^\"')]+)[\"']?\\)/i);
+      if (match) {
+        const url = resolveUrl(match[1], docUrl);
+        if (url) addImage(url, "", 0, 0, prefix + getSelector(el), "background-inline");
+      }
+    });
+
+    // 4. <style> tag CSS parsing
+    root.querySelectorAll("style").forEach((styleTag) => {
+      const css = styleTag.textContent || "";
+      const regex = /background-image\\s*:\\s*url\\([\"']?([^\"')]+)[\"']?\\)/gi;
+      let m;
+      while ((m = regex.exec(css)) !== null) {
+        const url = resolveUrl(m[1], docUrl);
+        if (url) addImage(url, "", 0, 0, prefix + "<style>", "background-css");
+      }
+    });
+
+    // 5. JSON-LD
+    root.querySelectorAll('script[type="application/ld+json"]').forEach((script) => {
+      try {
+        const data = JSON.parse(script.textContent || "{}");
+        const extractImages = (obj) => {
+          if (!obj) return;
+          if (typeof obj === "string" && /^https?:\\/\\//.test(obj) && /\\.(jpg|jpeg|png|gif|webp|svg|avif|bmp)(\\?|$)/i.test(obj)) {
+            addImage(obj, "", 0, 0, prefix + "<json-ld>", "jsonld");
+            return;
+          }
+          if (Array.isArray(obj)) { obj.forEach(extractImages); return; }
+          if (typeof obj === "object") {
+            if (obj.image) extractImages(obj.image);
+            if (obj.images) extractImages(obj.images);
+            if (obj.thumbnailUrl) extractImages(obj.thumbnailUrl);
+            if (obj.contentUrl && /\\.(jpg|jpeg|png|gif|webp|svg|avif|bmp)(\\?|$)/i.test(obj.contentUrl)) {
+              addImage(obj.contentUrl, obj.name || "", 0, 0, prefix + "<json-ld>", "jsonld");
+            }
+            Object.values(obj).forEach((v) => {
+              if (typeof v === "string" && /^https?:\\/\\//.test(v) && /\\.(jpg|jpeg|png|gif|webp|svg|avif|bmp)(\\?|$)/i.test(v)) {
+                addImage(v, "", 0, 0, prefix + "<json-ld>", "jsonld");
+              }
+            });
+          }
+        };
+        extractImages(data);
+      } catch {}
+    });
+
+    // 6. Inline scripts — window.__INITIAL_STATE__, __DATA__, etc.
+    root.querySelectorAll("script:not([src])").forEach((script) => {
+      const text = script.textContent || "";
+      const regex = /https?:\\/\\/[^\\s\"'<>]+\\.(jpg|jpeg|png|gif|webp|svg|avif|bmp)(\\?[^\\s\"'<>]*)?/gi;
+      let m;
+      while ((m = regex.exec(text)) !== null) {
+        addImage(m[0], "", 0, 0, prefix + "<inline-script>", "inline-script");
+      }
+    });
+
+    // 7. Shadow DOM recursion
+    root.querySelectorAll("*").forEach((el) => {
+      if (el.shadowRoot) {
+        extractFromRoot(el.shadowRoot, docUrl, prefix + getSelector(el) + "::shadow ");
+      }
+    });
+  }
+
+  function getSelector(el) {
+    if (el.id) return "#" + el.id;
+    let path = el.tagName.toLowerCase();
+    if (el.className && typeof el.className === "string") {
+      path += "." + el.className.split(/\\s+/).filter(Boolean).join(".");
+    }
+    return path;
+  }
+
+  // Main extraction
+  extractFromRoot(document, document.location.href, "");
+
+  return results;
+})()
+`;
+// ── Auto-click script (runs inside page.evaluate) ──
+const AUTOCLICK_SCRIPT = `
+(() => {
+  const keywords = [
+    "show more", "load more", "gallery", "view images", "view photos",
+    "prikaži više", "učitaj još", "galerija", "slike", "fotografije",
+    "photos", "images", "see more", "more images", "more photos",
+    "expand", "prikazi jos", "jos slika", "vise slika",
+  ];
+  const clicked = [];
+  const buttons = Array.from(document.querySelectorAll("button, a, [role=button], .btn, .button, [class*=gallery], [class*=image], [class*=photo], [class*=more], [class*=load]"));
+  for (const btn of buttons) {
+    const text = (btn.textContent || btn.title || btn.getAttribute("aria-label") || "").toLowerCase();
+    const found = keywords.some((k) => text.includes(k));
+    if (found && btn.offsetParent !== null) {
+      try {
+        btn.click();
+        btn.scrollIntoView({ behavior: "instant", block: "center" });
+        clicked.push(text.slice(0, 100));
+      } catch {}
+    }
+  }
+  return clicked;
+})()
+`;
+// ── Node.js API ──
+export async function extractImagesFromPage(page, // Playwright Page
+options = {}) {
+    const { scrollFull = false, autoClick = false, iframeScan = false, waitForSelector, minWidth = 0, maxWidth = 99999, limit = 500, } = options;
+    // Wait for selector if requested
+    if (waitForSelector) {
+        try {
+            await page.waitForSelector(waitForSelector, { timeout: 10_000 });
+        }
+        catch {
+            debugLog("image-extract", `waitForSelector "${waitForSelector}" timed out, continuing anyway`);
+        }
+    }
+    // Scroll full page to trigger lazy loading
+    if (scrollFull) {
+        await scrollFullPage(page);
+    }
+    // Auto-click "load more" buttons
+    if (autoClick) {
+        for (let i = 0; i < 5; i++) {
+            const clicked = await page.evaluate(AUTOCLICK_SCRIPT);
+            if (clicked.length === 0)
+                break;
+            debugLog("image-extract", `Auto-clicked: ${clicked.join(", ")}`);
+            await page.waitForTimeout(2500);
+            if (scrollFull)
+                await scrollFullPage(page);
+        }
+    }
+    // Main extraction
+    let images = await page.evaluate(EXTRACTION_SCRIPT);
+    // Iframe scan (same-origin only)
+    if (iframeScan) {
+        const iframeImages = await extractFromIframes(page, options);
+        images = images.concat(iframeImages);
+    }
+    // Deduplicate by URL
+    const unique = new Map();
+    for (const img of images) {
+        if (!unique.has(img.url))
+            unique.set(img.url, img);
+    }
+    images = Array.from(unique.values());
+    // Filter by dimensions
+    images = images.filter((img) => img.width >= minWidth && img.width <= maxWidth);
+    // Limit
+    if (limit > 0 && images.length > limit) {
+        images = images.slice(0, limit);
+    }
+    debugLog("image-extract", `Discovered ${images.length} unique images`);
+    return images;
+}
+async function scrollFullPage(page) {
+    await page.evaluate(async () => {
+        const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+        const height = document.body.scrollHeight;
+        const viewport = window.innerHeight;
+        let current = 0;
+        while (current < height) {
+            current += viewport;
+            window.scrollTo(0, current);
+            await delay(800);
+        }
+        window.scrollTo(0, 0);
+        await delay(500);
+    });
+}
+async function extractFromIframes(page, options) {
+    const frames = page.frames();
+    const results = [];
+    for (const frame of frames) {
+        if (frame === page.mainFrame())
+            continue;
+        try {
+            const url = frame.url();
+            if (!url || url === "about:blank")
+                continue;
+            // Only same-origin frames (cross-origin frames block evaluate)
+            const frameImages = await frame.evaluate(EXTRACTION_SCRIPT);
+            for (const img of frameImages) {
+                img.source = "iframe";
+                results.push(img);
+            }
+        }
+        catch {
+            // Cross-origin or detached frame — skip
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=browser-image-extract.js.map

package/dist/stealth/browser-image-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-image-extract.js","sourceRoot":"","sources":["../../src/stealth/browser-image-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAmC7C,mEAAmE;AAEnE,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqIzB,CAAC;AAEF,sDAAsD;AAEtD,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;CAuBxB,CAAC;AAEF,oBAAoB;AAEpB,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAS,EAAE,kBAAkB;AAC7B,UAAkC,EAAE;IAEpC,MAAM,EACJ,UAAU,GAAG,KAAK,EAClB,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,eAAe,EACf,QAAQ,GAAG,CAAC,EACZ,QAAQ,GAAG,KAAK,EAChB,KAAK,GAAG,GAAG,GACZ,GAAG,OAAO,CAAC;IAEZ,iCAAiC;IACjC,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,eAAe,EAAE,oBAAoB,eAAe,gCAAgC,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,iCAAiC;IACjC,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAChC,QAAQ,CAAC,eAAe,EAAE,iBAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjE,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,UAAU;gBAAE,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAEtE,iCAAiC;IACjC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACvC,CAAC;IAED,qBAAqB;IACrB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,IAAI,QAAQ,IAAI,GAAG,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;IAEhF,QAAQ;IACR,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QACvC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,QAAQ,CAAC,eAAe,EAAE,cAAc,MAAM,CAAC,MAAM,gBAAgB,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAS;IACrC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE;QAC7B,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;QACpC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,OAAO,GAAG,MAAM,EAAE,CAAC;YACxB,OAAO,IAAI,QAAQ,CAAC;YACpB,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAC5B,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtB,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,IAAS,EACT,OAA+B;IAE/B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,SAAS;QACzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,aAAa;gBAAE,SAAS;YAC5C,+DAA+D;YAC/D,MAAM,WAAW,GAAqB,MAAM,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAC9E,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/stealth/browser.d.ts

@@ -1,3 +1,4 @@
+import type { StoredCookie } from "../sessions/types.js";
 export interface BrowserFetchResult {
     html: string;
     status: number;
@@ -10,6 +11,10 @@ export interface BrowserFetchOptions {
     solveCaptcha?: boolean;
     proxyUrl?: string;
     chromeProfile?: string;
+    /** Cookies to inject into browser context before navigation */
+    cookies?: StoredCookie[];
+    /** Block images/fonts/css/tracking for faster loads (default: true for scrape, false for screenshot) */
+    blockResources?: boolean;
 }
 export declare function isPlaywrightAvailable(): Promise<boolean>;
 export declare function browserFetch(url: string, options?: BrowserFetchOptions): Promise<BrowserFetchResult & {

package/dist/stealth/browser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/stealth/browser.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAeD,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC,CAE9D;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,kBAAkB,GAAG;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,CAiEhF"}
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/stealth/browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AA2BzD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,wGAAwG;IACxG,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAeD,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC,CAE9D;AA4BD,wBAAsB,YAAY,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,kBAAkB,GAAG;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,CAiHhF"}

package/dist/stealth/browser.js

@@ -2,6 +2,29 @@ import { getTwoCaptchaApiKey } from "../config.js";
 import { HUMAN_DELAY_MIN_MS, HUMAN_DELAY_MAX_MS } from "../constants.js";
 import { trySolveCaptcha, hasCaptcha } from "../captcha/index.js";
 import { acquirePage } from "./chrome-profile.js";
+// ── Known tracking/ad domains to block for faster page loads ──
+const BLOCKED_DOMAINS = [
+    "google-analytics.com", "googletagmanager.com", "googlesyndication.com",
+    "doubleclick.net", "google-analytics.com", "googleadservices.com",
+    "facebook.com", "facebook.net", "fbcdn.net",
+    "twitter.com", "t.co",
+    "linkedin.com",
+    "analytics.tiktok.com",
+    "snap.licdn.com",
+    "bat.bing.com", "clarity.ms",
+    "hotjar.com", "mouseflow.com", "fullstory.com",
+    "segment.io", "segment.com",
+    "mixpanel.com", "amplitude.com", "heap.io",
+    "sentry.io", "bugsnag.com",
+    "newrelic.com", "nr-data.net",
+    "optimizely.com",
+    "criteo.com", "criteo.net",
+    "outbrain.com", "taboola.com",
+    "adnxs.com", "rubiconproject.com",
+    "moatads.com", "quantserve.com",
+];
+// Resource types to block for scraping (not screenshots)
+const BLOCKED_RESOURCE_TYPES = new Set(["image", "font", "stylesheet", "media"]);
 let playwrightAvailable = null;
 async function checkPlaywright() {
     if (playwrightAvailable !== null)
@@ -18,6 +41,22 @@ async function checkPlaywright() {
 export async function isPlaywrightAvailable() {
     return checkPlaywright();
 }
+/**
+ * Convert StoredCookie[] to Playwright's cookie format for injection.
+ */
+function toPlaywrightCookies(cookies, url) {
+    const parsed = new URL(url);
+    return cookies.map((c) => ({
+        name: c.name,
+        value: c.value,
+        domain: c.domain || parsed.hostname,
+        path: c.path || "/",
+        expires: c.expires,
+        httpOnly: c.httpOnly,
+        secure: c.secure ?? parsed.protocol === "https:",
+        sameSite: c.sameSite,
+    }));
+}
 export async function browserFetch(url, options) {
     const available = await checkPlaywright();
     if (!available) {
@@ -28,7 +67,35 @@ export async function browserFetch(url, options) {
         proxyUrl: options?.proxyUrl,
     });
     try {
-        const { page } = handle;
+        const { page, context } = handle;
+        // ── Inject accumulated cookies before navigation ──
+        if (options?.cookies && options.cookies.length > 0) {
+            const pwCookies = toPlaywrightCookies(options.cookies, url);
+            await context.addCookies(pwCookies);
+        }
+        // ── Resource blocking for faster scraping ──
+        const shouldBlockResources = options?.blockResources ?? !options?.screenshot;
+        if (shouldBlockResources) {
+            await page.route("**/*", (route) => {
+                const resourceType = route.request().resourceType();
+                const requestUrl = route.request().url();
+                // Block heavy resource types
+                if (BLOCKED_RESOURCE_TYPES.has(resourceType)) {
+                    return route.abort();
+                }
+                // Block known tracking domains
+                try {
+                    const reqHost = new URL(requestUrl).hostname;
+                    if (BLOCKED_DOMAINS.some((d) => reqHost.endsWith(d))) {
+                        return route.abort();
+                    }
+                }
+                catch {
+                    // Invalid URL, let it through
+                }
+                return route.continue();
+            });
+        }
         // Use "load" instead of "networkidle" — streaming sites (BBC, Binance, etc.)
         // never reach networkidle. After load, we briefly wait for networkidle as a bonus
         // but don't fail if it doesn't happen.
@@ -41,6 +108,20 @@ export async function browserFetch(url, options) {
             page.waitForLoadState("networkidle").catch(() => { }),
             new Promise((r) => setTimeout(r, 5000)),
         ]);
+        // ── SPA content-readiness check ──
+        // Poll for meaningful content (>200 chars text in body) up to 10s for SPAs
+        try {
+            await page.waitForFunction(() => {
+                const body = document.body;
+                if (!body)
+                    return false;
+                const text = body.innerText || "";
+                return text.length > 200;
+            }, { timeout: 10_000 }).catch(() => { });
+        }
+        catch {
+            // Content readiness check timed out — proceed with what we have
+        }
         // ── Human-like delay — anti-bot systems flag instant page reads ──
         const humanDelay = HUMAN_DELAY_MIN_MS + Math.random() * (HUMAN_DELAY_MAX_MS - HUMAN_DELAY_MIN_MS);
         await new Promise((r) => setTimeout(r, humanDelay));

package/dist/stealth/browser.js.map

@@ -1 +1 @@
-{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/stealth/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAiBlD,IAAI,mBAAmB,GAAmB,IAAI,CAAC;AAE/C,KAAK,UAAU,eAAe;IAC5B,IAAI,mBAAmB,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACrC,mBAAmB,GAAG,IAAI,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,mBAAmB,GAAG,KAAK,CAAC;IAC9B,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,eAAe,EAAE,CAAC;IAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IACxG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,OAAO,EAAE,aAAa;QACrC,QAAQ,EAAE,OAAO,EAAE,QAAQ;KAC5B,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,6EAA6E;QAC7E,kFAAkF;QAClF,uCAAuC;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACpC,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM;SACpC,CAAC,CAAC;QAEH,qFAAqF;QACrF,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;SACxC,CAAC,CAAC;QAEH,oEAAoE;QACpE,MAAM,UAAU,GAAG,kBAAkB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC;QAClG,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAEpD,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,6BAA6B;QAC7B,sEAAsE;QACtE,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,YAAY,KAAK,KAAK,IAAI,CAAC,CAAC,aAAa,CAAC;QAE9E,IAAI,kBAAkB,IAAI,aAAa,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;gBAC3D,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,gBAAgB,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAG;YACjC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,UAAU,EAAE,gBAAgB;YAC5B,aAAa;YACb,aAAa,EAAE,MAAM,CAAC,WAAW;SAClC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/stealth/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGlD,iEAAiE;AACjE,MAAM,eAAe,GAAG;IACtB,sBAAsB,EAAE,sBAAsB,EAAE,uBAAuB;IACvE,iBAAiB,EAAE,sBAAsB,EAAE,sBAAsB;IACjE,cAAc,EAAE,cAAc,EAAE,WAAW;IAC3C,aAAa,EAAE,MAAM;IACrB,cAAc;IACd,sBAAsB;IACtB,gBAAgB;IAChB,cAAc,EAAE,YAAY;IAC5B,YAAY,EAAE,eAAe,EAAE,eAAe;IAC9C,YAAY,EAAE,aAAa;IAC3B,cAAc,EAAE,eAAe,EAAE,SAAS;IAC1C,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,aAAa;IAC7B,gBAAgB;IAChB,YAAY,EAAE,YAAY;IAC1B,cAAc,EAAE,aAAa;IAC7B,WAAW,EAAE,oBAAoB;IACjC,aAAa,EAAE,gBAAgB;CAChC,CAAC;AAEF,yDAAyD;AACzD,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;AAqBjF,IAAI,mBAAmB,GAAmB,IAAI,CAAC;AAE/C,KAAK,UAAU,eAAe;IAC5B,IAAI,mBAAmB,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACrC,mBAAmB,GAAG,IAAI,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,mBAAmB,GAAG,KAAK,CAAC;IAC9B,CAAC;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,OAAuB,EAAE,GAAW;IAU/D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ;QACnC,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG;QACnB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAChD,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,eAAe,EAAE,CAAC;IAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IACxG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,OAAO,EAAE,aAAa;QACrC,QAAQ,EAAE,OAAO,EAAE,QAAQ;KAC5B,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAEjC,qDAAqD;QACrD,IAAI,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,MAAM,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC5D,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QAED,8CAA8C;QAC9C,MAAM,oBAAoB,GAAG,OAAO,EAAE,cAAc,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC;QAC7E,IAAI,oBAAoB,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAsI,EAAE,EAAE;gBAClK,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC;gBAEzC,6BAA6B;gBAC7B,IAAI,sBAAsB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC7C,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;gBAED,+BAA+B;gBAC/B,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC;oBAC7C,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBACrD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;oBACvB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,8BAA8B;gBAChC,CAAC;gBAED,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC1B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,6EAA6E;QAC7E,kFAAkF;QAClF,uCAAuC;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACpC,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM;SACpC,CAAC,CAAC;QAEH,qFAAqF;QACrF,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;SACxC,CAAC,CAAC;QAEH,oCAAoC;QACpC,2EAA2E;QAC3E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;gBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,IAAI,CAAC,IAAI;oBAAE,OAAO,KAAK,CAAC;gBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;gBAClC,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;YAC3B,CAAC,EACD,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACpB,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;QAED,oEAAoE;QACpE,MAAM,UAAU,GAAG,kBAAkB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC;QAClG,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QAEpD,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,6BAA6B;QAC7B,sEAAsE;QACtE,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAC;QAC5C,MAAM,kBAAkB,GAAG,OAAO,EAAE,YAAY,KAAK,KAAK,IAAI,CAAC,CAAC,aAAa,CAAC;QAE9E,IAAI,kBAAkB,IAAI,aAAa,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;gBAC3D,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,gBAAgB,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAG;YACjC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,UAAU,EAAE,gBAAgB;YAC5B,aAAa;YACb,aAAa,EAAE,MAAM,CAAC,WAAW;SAClC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}

package/dist/stealth/chrome-profile.d.ts

@@ -20,6 +20,7 @@ export interface PageHandle {
 export interface AcquirePageOptions {
     chromeProfile?: string;
     proxyUrl?: string;
+    headless?: boolean;
 }
 /**
  * Resolve which Chrome profile to use.

package/dist/stealth/chrome-profile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chrome-profile.d.ts","sourceRoot":"","sources":["../../src/stealth/chrome-profile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,KAAK,IAAI,GAAG,OAAO,sBAAsB,EAAE,IAAI,CAAC;AAChD,KAAK,cAAc,GAAG,OAAO,sBAAsB,EAAE,cAAc,CAAC;AAGpE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG1E;AA6HD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,UAAU,CAAC,CAUnF"}
+{"version":3,"file":"chrome-profile.d.ts","sourceRoot":"","sources":["../../src/stealth/chrome-profile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,KAAK,IAAI,GAAG,OAAO,sBAAsB,EAAE,IAAI,CAAC;AAChD,KAAK,cAAc,GAAG,OAAO,sBAAsB,EAAE,cAAc,CAAC;AAGpE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG1E;AAqJD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,UAAU,CAAC,CAUnF"}

package/dist/stealth/chrome-profile.js

@@ -46,11 +46,11 @@ async function withProfileLock(profilePath, fn) {
     }
 }
 // ── Profile Page Acquisition ──
-async function acquireProfilePage(profilePath, proxyUrl) {
+async function acquireProfilePage(profilePath, proxyUrl, headless = true) {
     const { chromium } = await import("rebrowser-playwright");
     const { timeZone, locale } = Intl.DateTimeFormat().resolvedOptions();
     const launchOptions = {
-        headless: true,
+        headless,
         args: STEALTH_ARGS,
         timezoneId: timeZone,
         locale,
@@ -82,7 +82,30 @@ async function acquireProfilePage(profilePath, proxyUrl) {
     };
 }
 // ── Pool Page Acquisition (existing behavior) ──
-async function acquirePoolPage(proxyUrl) {
+async function acquirePoolPage(proxyUrl, headless = true) {
+    if (!headless) {
+        const { chromium } = await import("rebrowser-playwright");
+        const { timeZone, locale } = Intl.DateTimeFormat().resolvedOptions();
+        const browser = await chromium.launch({
+            headless: false,
+            args: STEALTH_ARGS,
+            ...(proxyUrl && { proxy: { server: proxyUrl } }),
+        });
+        const context = await browser.newContext({
+            viewport: DEFAULT_VIEWPORT,
+            timezoneId: timeZone,
+            locale,
+        });
+        const page = await context.newPage();
+        return {
+            page,
+            context,
+            isProfile: false,
+            cleanup: async () => {
+                await browser.close();
+            },
+        };
+    }
     const pool = getPool();
     const browser = await pool.acquire(proxyUrl);
     let page;
@@ -133,8 +156,8 @@ async function acquirePoolPage(proxyUrl) {
 export async function acquirePage(options) {
     const profilePath = resolveChromeProfile(options?.chromeProfile);
     if (profilePath) {
-        return withProfileLock(profilePath, () => acquireProfilePage(profilePath, options?.proxyUrl));
+        return withProfileLock(profilePath, () => acquireProfilePage(profilePath, options?.proxyUrl, options?.headless ?? true));
     }
-    return acquirePoolPage(options?.proxyUrl);
+    return acquirePoolPage(options?.proxyUrl, options?.headless ?? true);
 }
 //# sourceMappingURL=chrome-profile.js.map

package/dist/stealth/chrome-profile.js.map

@@ -1 +1 @@
-{"version":3,"file":"chrome-profile.js","sourceRoot":"","sources":["../../src/stealth/chrome-profile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAmB5C;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAiB;IACpD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,OAAO,oBAAoB,EAAE,CAAC;AAChC,CAAC;AAED,sBAAsB;AACtB,yEAAyE;AAEzE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;AAEtD,KAAK,UAAU,eAAe,CAAI,WAAmB,EAAE,EAAoB;IACzE,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE3D,wEAAwE;IACxE,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;IAElE,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC;IACzB,CAAC;YAAS,CAAC;QACT,iCAAiC;QACjC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,6CAA6C;gBAC7C,IAAI,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,OAAO,EAAE,CAAC;oBAC9C,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC;AAED,iCAAiC;AAEjC,KAAK,UAAU,kBAAkB,CAAC,WAAmB,EAAE,QAAiB;IACtE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAE1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC;IAErE,MAAM,aAAa,GAA4B;QAC7C,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,YAAY;QAClB,UAAU,EAAE,QAAQ;QACpB,MAAM;KACP,CAAC;IAEF,IAAI,QAAQ,EAAE,CAAC;QACb,aAAa,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC7C,CAAC;IAED,IAAI,OAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,kEAAkE;QAClE,OAAO,GAAG,MAAM,QAAQ,CAAC,uBAAuB,CAAC,WAAW,EAAE;YAC5D,GAAG,aAAa;YAChB,OAAO,EAAE,QAAQ;SACwC,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,4CAA4C;QAC5C,OAAO,GAAG,MAAM,QAAQ,CAAC,uBAAuB,CAC9C,WAAW,EACX,aAAuE,CACxE,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO;QACL,IAAI;QACJ,OAAO;QACP,SAAS,EAAE,IAAI;QACf,WAAW;QACX,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kDAAkD;AAElD,KAAK,UAAU,eAAe,CAAC,QAAiB;IAC9C,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,IAAU,CAAC;IACf,IAAI,SAAqC,CAAC;IAE1C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC;IAErE,IAAI,CAAC;QACH,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACpE,SAAS,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE;YAC5C,iBAAiB,EAAE;gBACjB,UAAU,EAAE,QAAQ;gBACpB,MAAM;aACP;SACF,CAAC,CAAC;QACH,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;QAC3D,OAAO,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;QACnG,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACnC,QAAQ,EAAE,gBAAgB;YAC1B,UAAU,EAAE,QAAQ;YACpB,MAAM;SACP,CAAC,CAAC;QACH,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,OAAO;QACL,IAAI;QACJ,OAAO;QACP,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,yBAAyB;AAEzB;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA4B;IAC5D,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEjE,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,eAAe,CAAC,WAAW,EAAE,GAAG,EAAE,CACvC,kBAAkB,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,CACnD,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"chrome-profile.js","sourceRoot":"","sources":["../../src/stealth/chrome-profile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAoB5C;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAiB;IACpD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,OAAO,oBAAoB,EAAE,CAAC;AAChC,CAAC;AAED,sBAAsB;AACtB,yEAAyE;AAEzE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;AAEtD,KAAK,UAAU,eAAe,CAAI,WAAmB,EAAE,EAAoB;IACzE,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE3D,wEAAwE;IACxE,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;IAElE,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC;IACzB,CAAC;YAAS,CAAC;QACT,iCAAiC;QACjC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,6CAA6C;gBAC7C,IAAI,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,OAAO,EAAE,CAAC;oBAC9C,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC;AAED,iCAAiC;AAEjC,KAAK,UAAU,kBAAkB,CAAC,WAAmB,EAAE,QAAiB,EAAE,QAAQ,GAAG,IAAI;IACvF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAE1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC;IAErE,MAAM,aAAa,GAA4B;QAC7C,QAAQ;QACR,IAAI,EAAE,YAAY;QAClB,UAAU,EAAE,QAAQ;QACpB,MAAM;KACP,CAAC;IAEF,IAAI,QAAQ,EAAE,CAAC;QACb,aAAa,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC7C,CAAC;IAED,IAAI,OAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,kEAAkE;QAClE,OAAO,GAAG,MAAM,QAAQ,CAAC,uBAAuB,CAAC,WAAW,EAAE;YAC5D,GAAG,aAAa;YAChB,OAAO,EAAE,QAAQ;SACwC,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,4CAA4C;QAC5C,OAAO,GAAG,MAAM,QAAQ,CAAC,uBAAuB,CAC9C,WAAW,EACX,aAAuE,CACxE,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAE3D,OAAO;QACL,IAAI;QACJ,OAAO;QACP,SAAS,EAAE,IAAI;QACf,WAAW;QACX,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,kDAAkD;AAElD,KAAK,UAAU,eAAe,CAAC,QAAiB,EAAE,QAAQ,GAAG,IAAI;IAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC;QACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACpC,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,YAAY;YAClB,GAAG,CAAC,QAAQ,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;SACjD,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,QAAQ,EAAE,gBAAgB;YAC1B,UAAU,EAAE,QAAQ;YACpB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,OAAO;YACP,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,KAAK,IAAI,EAAE;gBAClB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,IAAU,CAAC;IACf,IAAI,SAAqC,CAAC;IAE1C,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,eAAe,EAAE,CAAC;IAErE,IAAI,CAAC;QACH,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACpE,SAAS,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE;YAC5C,iBAAiB,EAAE;gBACjB,UAAU,EAAE,QAAQ;gBACpB,MAAM;aACP;SACF,CAAC,CAAC;QACH,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;QAC3D,OAAO,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;QACnG,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACnC,QAAQ,EAAE,gBAAgB;YAC1B,UAAU,EAAE,QAAQ;YACpB,MAAM;SACP,CAAC,CAAC;QACH,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,OAAO;QACL,IAAI;QACJ,OAAO;QACP,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,yBAAyB;AAEzB;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA4B;IAC5D,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEjE,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,eAAe,CAAC,WAAW,EAAE,GAAG,EAAE,CACvC,kBAAkB,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,CAC9E,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAC;AACvE,CAAC"}

package/dist/stealth/detector.d.ts

@@ -1,8 +1,17 @@
 /**
  * Detect if a response is blocked by anti-bot protection.
- * Checks HTTP status, page content, and structural signals.
+ *
+ * Uses a layered approach to minimize false positives:
+ * 1. Status codes (403/429/503) only trigger block if paired with anti-bot signals
+ * 2. Content-based checks only run on small pages (<5KB text)
+ * 3. Large pages with navigation are treated as legitimate regardless
  */
 export declare function isBlocked(html: string, status: number, headers?: Record<string, string>): boolean;
+/**
+ * Detect if a page contains a CAPTCHA challenge.
+ * Separate from isBlocked — a CAPTCHA page CAN be solved, it's not a hard block.
+ */
+export declare function isCaptchaPage(html: string): boolean;
 /**
  * Detect if a page requires JavaScript rendering to get content.
  */

package/dist/stealth/detector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../../src/stealth/detector.ts"],"names":[],"mappings":"AA6CA;;;GAGG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAiCjG;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAUtD"}
+{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../../src/stealth/detector.ts"],"names":[],"mappings":"AAuGA;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CA4DjG;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAUnD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAUtD"}