@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/browser-fetch.js

@@ -0,0 +1,1600 @@
+/**
+ * Browser-based fetching — uses Playwright via the browser pool.
+ * Handles browserFetch, browserScreenshot, retryFetch, and scrollAndWait.
+ */
+import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types.js';
+import { detectChallenge } from './challenge-detection.js';
+import { getRealisticUserAgent } from './user-agents.js';
+import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getStealthPlaywright, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, ANTI_DETECTION_ARGS, getRandomViewport, } from './browser-pool.js';
+// Proprietary stealth module — gitignored, loaded conditionally
+let applyStealthPatches;
+let applyAcceptLanguageHeader;
+try {
+    // @ts-ignore — proprietary module, gitignored
+    const mod = await import('./stealth-patches.js');
+    applyStealthPatches = mod.applyStealthPatches;
+    applyAcceptLanguageHeader = mod.applyAcceptLanguageHeader;
+}
+catch { /* Not available in CI/open-source builds */ }
+import { validateUrl, validateUserAgent, createAbortError } from './http-fetch.js';
+import { autoInteract } from './auto-interact.js';
+import { createLogger } from './logger.js';
+const log = createLogger('browser');
+// ── Execution context error detection ─────────────────────────────────────────
+/**
+ * Check if an error indicates the page execution context was destroyed.
+ * This happens on SPAs (like Polymarket) when scrolling triggers navigation.
+ */
+function isContextDestroyedError(err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return (msg.includes('Execution context was destroyed') ||
+        msg.includes('Target closed') ||
+        msg.includes('frame was detached') ||
+        msg.includes('Session closed'));
+}
+/**
+ * Resilient scrollThrough helper — scrolls from top to bottom to trigger
+ * IntersectionObservers, lazy loading, and animations.
+ *
+ * Handles SPAs gracefully: if the execution context is destroyed during scroll
+ * (e.g., Polymarket navigation), logs a warning and stops — does NOT throw.
+ *
+ * Also tries inner scrollable containers (virtual scroll) before falling back
+ * to window.scrollTo.
+ */
+async function resilientScrollThrough(page, delayMs = 250) {
+    try {
+        // Use shared container detection from actions module
+        const { detectScrollContainer } = await import('./actions.js');
+        const containerSelector = (await detectScrollContainer(page)) || null;
+        if (containerSelector) {
+            // Scroll inner container
+            try {
+                const scrollHeight = await page.evaluate((sel) => {
+                    const el = document.querySelector(sel);
+                    return el ? el.scrollHeight : document.body.scrollHeight;
+                }, containerSelector);
+                const vh = await page.evaluate(() => window.innerHeight);
+                for (let y = 0; y < scrollHeight; y += Math.round(vh * 0.75)) {
+                    await page.evaluate(([sel, sy]) => {
+                        const el = document.querySelector(sel);
+                        if (el)
+                            el.scrollTop = sy;
+                    }, [containerSelector, y]);
+                    await page.waitForTimeout(delayMs);
+                }
+                await page.evaluate((sel) => {
+                    const el = document.querySelector(sel);
+                    if (el)
+                        el.scrollTop = el.scrollHeight;
+                }, containerSelector);
+                await page.waitForTimeout(Math.round(delayMs * 1.6));
+                await page.evaluate((sel) => {
+                    const el = document.querySelector(sel);
+                    if (el)
+                        el.scrollTop = 0;
+                }, containerSelector);
+                await page.waitForTimeout(Math.round(delayMs * 2.4));
+                return;
+            }
+            catch (innerErr) {
+                if (isContextDestroyedError(innerErr)) {
+                    log.warn('Execution context destroyed during inner container scroll — continuing with captured content');
+                    return;
+                }
+                // Fall through to window scroll
+            }
+        }
+        // Window-level scroll (standard path)
+        const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
+        const viewportHeight = await page.evaluate(() => window.innerHeight);
+        for (let y = 0; y < scrollHeight; y += Math.round(viewportHeight * 0.75)) {
+            await page.evaluate((sy) => window.scrollTo({ top: sy, behavior: 'instant' }), y);
+            await page.waitForTimeout(delayMs);
+        }
+        await page.evaluate(() => window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }));
+        await page.waitForTimeout(Math.round(delayMs * 1.6));
+        await page.evaluate(() => window.scrollTo({ top: 0, behavior: 'instant' }));
+        await page.waitForTimeout(Math.round(delayMs * 2.4));
+    }
+    catch (err) {
+        if (isContextDestroyedError(err)) {
+            log.warn('Execution context destroyed during scrollThrough — continuing with captured content');
+            return;
+        }
+        // Re-throw unexpected errors
+        throw err;
+    }
+}
+// ── Concurrency state (owned by this module) ─────────────────────────────────
+let activePagesCount = 0;
+// ── browserFetch ──────────────────────────────────────────────────────────────
+/**
+ * Fetch using headless Chromium via Playwright
+ * Slower but can handle JavaScript-heavy sites and bypass some bot detection
+ */
+export async function browserFetch(url, options = {}) {
+    // SECURITY: Validate URL to prevent SSRF
+    validateUrl(url);
+    const { userAgent, waitMs = 0, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies, stealth = false, actions, keepPageOpen = false, signal, profileDir, headed = false, storageState, proxy, device = 'desktop', viewportWidth: optViewportWidth, viewportHeight: optViewportHeight, deviceScaleFactor: optDeviceScaleFactor, waitUntil: optWaitUntil, waitSelector, blockResources, isSPA = false, languages, } = options;
+    // Device emulation profiles (with deviceScaleFactor for crisp screenshots)
+    const deviceProfiles = {
+        desktop: { width: 1920, height: 1080, deviceScaleFactor: 1, userAgent: undefined },
+        mobile: {
+            width: 390,
+            height: 844,
+            deviceScaleFactor: 3,
+            userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+        },
+        tablet: {
+            width: 820,
+            height: 1180,
+            deviceScaleFactor: 2,
+            userAgent: 'Mozilla/5.0 (iPad; CPU OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+        },
+    };
+    const deviceProfile = deviceProfiles[device] ?? deviceProfiles.desktop;
+    const effectiveViewportWidth = optViewportWidth ?? deviceProfile.width;
+    const effectiveViewportHeight = optViewportHeight ?? deviceProfile.height;
+    const effectiveScaleFactor = optDeviceScaleFactor ?? deviceProfile.deviceScaleFactor;
+    const effectiveWaitUntil = optWaitUntil || 'domcontentloaded';
+    // Validate user agent if provided
+    // In stealth mode with no custom UA, always use a realistic Chrome UA
+    const validatedUserAgent = userAgent
+        ? validateUserAgent(userAgent)
+        : (stealth ? getRealisticUserAgent() : getRandomUserAgent());
+    // Validate wait time
+    if (waitMs < 0 || waitMs > 60000) {
+        throw new WebPeelError('Wait time must be between 0 and 60000ms');
+    }
+    if (signal?.aborted) {
+        throw createAbortError();
+    }
+    // SECURITY: Validate custom headers if provided
+    if (headers) {
+        for (const [key, value] of Object.entries(headers)) {
+            // Block Host header override
+            if (key.toLowerCase() === 'host') {
+                throw new WebPeelError('Custom Host header is not allowed');
+            }
+            if (typeof value !== 'string' || value.length > 500) {
+                throw new WebPeelError('Invalid header value');
+            }
+        }
+    }
+    // SECURITY: Limit concurrent browser pages with timeout
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000; // 30 second max wait
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    let usingPooledPage = false;
+    let abortHandler;
+    // Declared here (outside try) so the finally block can reference it
+    const usingProfileBrowser = !!profileDir;
+    // Owned context created when storageState injection is requested
+    let ownedContext;
+    // Owned browser launched when proxy is specified (dedicated browser with proxy at launch level)
+    let ownedBrowser;
+    try {
+        const browser = usingProfileBrowser
+            ? await getProfileBrowser(profileDir, headed, stealth)
+            : stealth
+                ? await getStealthBrowser()
+                : await getBrowser();
+        // Only use the shared page pool for non-stealth, non-profile, non-keepOpen, non-storageState, non-proxy fetches
+        const shouldUsePagePool = !stealth && !userAgent && !keepPageOpen && !usingProfileBrowser && !storageState && !proxy;
+        if (shouldUsePagePool) {
+            page = takePooledPage();
+            usingPooledPage = !!page;
+            if (usingPooledPage && getPooledPagesCount() < PAGE_POOL_SIZE) {
+                void ensurePagePool(browser).catch(() => { });
+            }
+        }
+        if (!page) {
+            const effectiveLocale = (languages && languages.length > 0) ? languages[0] : 'en-US';
+            const pageOptions = {
+                userAgent: validatedUserAgent,
+                // viewport: null lets the browser use its natural window size (set via --window-size),
+                // avoiding the telltale Playwright default of 1280×720.
+                viewport: null,
+                ...(stealth
+                    ? {
+                        locale: effectiveLocale,
+                        timezoneId: 'America/New_York',
+                        javaScriptEnabled: true,
+                    }
+                    : {}),
+            };
+            if (proxy) {
+                // Parse proxy URL to extract auth credentials for Playwright
+                let playwrightProxy;
+                try {
+                    const proxyUrl = new URL(proxy);
+                    playwrightProxy = {
+                        server: `${proxyUrl.protocol}//${proxyUrl.host}`,
+                        username: proxyUrl.username || undefined,
+                        password: proxyUrl.password || undefined,
+                    };
+                }
+                catch (e) {
+                    // Fallback: use proxy string as-is
+                    log.debug('proxy URL parse failed, using as-is:', e instanceof Error ? e.message : e);
+                    playwrightProxy = { server: proxy };
+                }
+                // Launch a DEDICATED fresh browser with proxy at the launch level.
+                // Context-level proxy is unreliable for anti-bot sites — they check the browser's
+                // IP at connection time (set at launch), not at context creation.
+                const pw = stealth ? await getStealthPlaywright() : (await import('playwright')).chromium;
+                const vp = getRandomViewport();
+                ownedBrowser = await pw.launch({
+                    headless: true,
+                    args: [...ANTI_DETECTION_ARGS, `--window-size=${vp.width},${vp.height}`],
+                    proxy: playwrightProxy,
+                });
+                ownedContext = await ownedBrowser.newContext({
+                    userAgent: validatedUserAgent || getRandomUserAgent(),
+                    locale: effectiveLocale,
+                    timezoneId: 'America/New_York',
+                    javaScriptEnabled: true,
+                    viewport: { width: effectiveViewportWidth || vp.width, height: effectiveViewportHeight || vp.height },
+                    deviceScaleFactor: effectiveScaleFactor,
+                    ...(storageState ? { storageState } : {}),
+                });
+                page = await ownedContext.newPage();
+            }
+            else if (storageState) {
+                // Create an isolated context with the injected storage state (cookies + localStorage)
+                ownedContext = await browser.newContext({
+                    ...pageOptions,
+                    storageState,
+                    viewport: { width: effectiveViewportWidth, height: effectiveViewportHeight },
+                    deviceScaleFactor: effectiveScaleFactor,
+                });
+                page = await ownedContext.newPage();
+            }
+            else {
+                // When deviceScaleFactor differs from default (1), create an isolated context
+                // so the scale factor is applied (Playwright requires it at context level)
+                if (effectiveScaleFactor !== 1) {
+                    ownedContext = await browser.newContext({
+                        ...pageOptions,
+                        viewport: { width: effectiveViewportWidth, height: effectiveViewportHeight },
+                        deviceScaleFactor: effectiveScaleFactor,
+                    });
+                    page = await ownedContext.newPage();
+                }
+                else {
+                    page = await browser.newPage(pageOptions);
+                    // Apply viewport for device emulation or explicit viewport overrides
+                    if (device !== 'desktop' || optViewportWidth !== undefined || optViewportHeight !== undefined) {
+                        await page.setViewportSize({ width: effectiveViewportWidth, height: effectiveViewportHeight }).catch(() => { });
+                    }
+                }
+            }
+            await applyStealthScripts(page, languages);
+            // Apply supplemental stealth patches (canvas noise, connection API, battery, etc.)
+            // These go beyond what puppeteer-extra-plugin-stealth provides.
+            if (stealth) {
+                await applyStealthPatches?.(page);
+                await applyAcceptLanguageHeader?.(page, 'en-US');
+            }
+            usingPooledPage = false;
+        }
+        else {
+            await page.setViewportSize({ width: effectiveViewportWidth, height: effectiveViewportHeight }).catch(() => { });
+        }
+        if (signal) {
+            abortHandler = () => {
+                if (page && !page.isClosed()) {
+                    void page.close().catch(() => { });
+                }
+            };
+            signal.addEventListener('abort', abortHandler, { once: true });
+        }
+        await page.unroute('**/*').catch(() => { });
+        const mergedHeaders = { ...(headers || {}) };
+        if (usingPooledPage) {
+            mergedHeaders['User-Agent'] = validatedUserAgent;
+        }
+        // Apply device user-agent (mobile/tablet) unless caller overrode userAgent
+        if (deviceProfile.userAgent && !userAgent) {
+            mergedHeaders['User-Agent'] = deviceProfile.userAgent;
+        }
+        if (usingPooledPage || Object.keys(mergedHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(mergedHeaders);
+        }
+        // Set cookies if provided
+        if (cookies && cookies.length > 0) {
+            const parsedCookies = cookies.map(cookie => {
+                const [nameValue] = cookie.split(';').map(s => s.trim());
+                const [name, value] = nameValue.split('=');
+                if (!name || value === undefined) {
+                    throw new WebPeelError(`Invalid cookie format: ${cookie}`);
+                }
+                return {
+                    name: name.trim(),
+                    value: value.trim(),
+                    url,
+                };
+            });
+            await page.context().addCookies(parsedCookies);
+        }
+        if (signal?.aborted) {
+            throw createAbortError();
+        }
+        // Block resources: custom list takes precedence; otherwise use defaults in non-screenshot/non-stealth mode.
+        // In stealth mode, blocking common resources can be a bot-detection signal.
+        if (blockResources && blockResources.length > 0) {
+            const blockedTypes = new Set(blockResources);
+            await page.route('**/*', (route) => {
+                const resourceType = route.request().resourceType();
+                if (blockedTypes.has(resourceType)) {
+                    route.abort();
+                }
+                else {
+                    route.continue();
+                }
+            });
+        }
+        else if (screenshot) {
+            // Screenshots need all resources (images, CSS) for visual accuracy
+            await page.route('**/*', (route) => route.continue());
+        }
+        else {
+            // Default: block images/fonts/media for speed + bandwidth savings.
+            // In stealth mode, we keep stylesheets (blocking CSS is a bot signal)
+            // but still block images/fonts/media (saves ~70% bandwidth per page).
+            const blocklist = stealth
+                ? ['image', 'font', 'media'] // stealth: keep CSS, block heavy assets
+                : ['image', 'font', 'media', 'stylesheet']; // normal: block everything non-text
+            await page.route('**/*', (route) => {
+                const resourceType = route.request().resourceType();
+                if (blocklist.includes(resourceType)) {
+                    route.abort();
+                }
+                else {
+                    route.continue();
+                }
+            });
+        }
+        // SECURITY: Wrap entire operation in timeout
+        let screenshotBuffer;
+        const throwIfAborted = () => {
+            if (signal?.aborted) {
+                throw createAbortError();
+            }
+        };
+        const fetchPromise = (async () => {
+            let response;
+            try {
+                response = await page.goto(url, {
+                    waitUntil: effectiveWaitUntil,
+                    timeout: timeoutMs,
+                });
+            }
+            catch (gotoError) {
+                const msg = gotoError?.message || String(gotoError);
+                if (/net::ERR_HTTP2_PROTOCOL_ERROR/i.test(msg)) {
+                    throw new BlockedError(`Site blocked the request (HTTP/2 protocol error). The site likely has anti-bot protection. Try using stealth mode or a proxy.`);
+                }
+                if (/net::ERR_CONNECTION_REFUSED/i.test(msg)) {
+                    throw new NetworkError(`Connection refused by the server at ${url}. The server may be down or blocking your IP.`);
+                }
+                if (/net::ERR_CONNECTION_RESET/i.test(msg)) {
+                    throw new BlockedError(`Connection was reset by the server. This typically indicates anti-bot protection or IP blocking. Try using stealth mode or a different IP.`);
+                }
+                if (/net::ERR_SSL/i.test(msg)) {
+                    throw new NetworkError(`SSL/TLS error connecting to site. URL: ${url}`);
+                }
+                if (/net::ERR_NAME_NOT_RESOLVED/i.test(msg)) {
+                    throw new NetworkError(`Domain not found: ${url}`);
+                }
+                if (/net::ERR_CERT/i.test(msg)) {
+                    throw new NetworkError(`SSL certificate error for ${url}`);
+                }
+                if (/NS_ERROR_NET_RESET/i.test(msg)) {
+                    throw new NetworkError(`Connection reset (Firefox). The site may be blocking automated access. URL: ${url}`);
+                }
+                if (/timeout/i.test(msg)) {
+                    throw new TimeoutError(`Page load timed out after ${timeoutMs}ms: ${url}`);
+                }
+                if (/net::ERR_/i.test(msg)) {
+                    throw new NetworkError(`Browser network error: ${msg.match(/net::ERR_\w+/i)?.[0] || msg}`);
+                }
+                throw gotoError;
+            }
+            throwIfAborted();
+            // Wait for a specific CSS selector if requested
+            if (waitSelector) {
+                await page.waitForSelector(waitSelector, { timeout: timeoutMs }).catch(() => {
+                    log.debug(`waitSelector "${waitSelector}" not found within timeout`);
+                });
+                throwIfAborted();
+            }
+            // Quick check: if body text is very thin, wait for JS to render more content.
+            // Only adds latency when the page clearly hasn't loaded yet.
+            // eslint-disable-next-line @typescript-eslint/no-implied-eval
+            const bodyTextLength = await page.evaluate('document.body?.innerText?.trim().length || 0').catch(() => 0);
+            if (bodyTextLength < 500) {
+                await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => { });
+                throwIfAborted();
+            }
+            // DOM stability check: wait for SPA hydration to settle.
+            // Polls innerText length every 500ms — if still growing, keep waiting.
+            // SPAs (Kayak, Google Flights, Expedia) get a longer timeout to allow async data loads.
+            {
+                const stabilityStart = Date.now();
+                // SPA sites (Kayak, Google Flights, Expedia) need up to 12s for results to load.
+                // Normal rendered pages need just 3s extra.
+                const MAX_STABILITY_WAIT_MS = isSPA ? 12000 : 3000;
+                // SPA: must be stable for 2s (4 consecutive 500ms checks). Normal: 1s (2 checks).
+                const STABLE_CHECKS_REQUIRED = isSPA ? 4 : 2;
+                const POLL_INTERVAL_MS = 500;
+                const MIN_CONTENT_LENGTH = 200; // Don't consider near-empty pages stable
+                let prevLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
+                let stableCount = 0;
+                while (Date.now() - stabilityStart < MAX_STABILITY_WAIT_MS) {
+                    throwIfAborted();
+                    await page.waitForTimeout(POLL_INTERVAL_MS);
+                    const curLength = await page.evaluate('document.body?.innerText?.length || 0').catch(() => 0);
+                    if (curLength !== prevLength) {
+                        // Content changed — reset stability counter
+                        stableCount = 0;
+                    }
+                    else if (curLength >= MIN_CONTENT_LENGTH) {
+                        stableCount++;
+                        if (stableCount >= STABLE_CHECKS_REQUIRED)
+                            break; // stable long enough
+                    }
+                    prevLength = curLength;
+                }
+                if (isSPA) {
+                    log.debug(`SPA stability check: ${Date.now() - stabilityStart}ms, length=${prevLength}`);
+                }
+            }
+            const finalUrl = page.url();
+            const contentType = response?.headers()?.['content-type'] || '';
+            const contentTypeLower = contentType.toLowerCase();
+            const urlLower = finalUrl.toLowerCase();
+            const isPdf = contentTypeLower.includes('application/pdf') || urlLower.endsWith('.pdf');
+            const isDocx = contentTypeLower.includes('wordprocessingml.document') || urlLower.endsWith('.docx');
+            const isBinaryDoc = !!response && (isPdf || isDocx);
+            // Small randomized delay in stealth mode (simulate human behavior)
+            // Keep it short — enough to look human, not enough to kill latency
+            if (stealth) {
+                const extraDelayMs = 200 + Math.floor(Math.random() * 601);
+                await page.waitForTimeout(extraDelayMs);
+                throwIfAborted();
+            }
+            // Human-like delay for proxied requests (helps bypass bot detection on strict sites)
+            if (proxy) {
+                // Realistic human behavior to bypass behavioral analysis
+                const humanDelay = 800 + Math.random() * 1200;
+                await page.waitForTimeout(humanDelay);
+                throwIfAborted();
+                // Realistic mouse movement (simulate human cursor)
+                try {
+                    const vw = await page.evaluate(() => window.innerWidth);
+                    const vh = await page.evaluate(() => window.innerHeight);
+                    await page.mouse.move(100 + Math.random() * (vw - 200), 100 + Math.random() * (vh - 200), { steps: 5 + Math.floor(Math.random() * 10) });
+                    // Small scroll to trigger lazy-loaded content
+                    await page.evaluate(() => window.scrollBy(0, 200 + Math.random() * 400));
+                    await page.waitForTimeout(300 + Math.random() * 500);
+                    throwIfAborted();
+                    // Second mouse move
+                    await page.mouse.move(50 + Math.random() * (vw - 100), 50 + Math.random() * (vh - 100), { steps: 3 + Math.floor(Math.random() * 5) });
+                }
+                catch {
+                    // Non-fatal: mouse/scroll simulation failed
+                }
+                throwIfAborted();
+            }
+            // Wait for additional time if requested (for dynamic content / screenshots)
+            if (waitMs > 0) {
+                await page.waitForTimeout(waitMs);
+                throwIfAborted();
+            }
+            // Auto-interact: dismiss cookie banners, consent popups, overlays
+            // before content extraction. Runs before user-specified actions so
+            // that popups don't interfere with custom interactions.
+            let autoInteractResult;
+            if (!isBinaryDoc) {
+                try {
+                    autoInteractResult = await Promise.race([
+                        autoInteract(page),
+                        new Promise((resolve) => setTimeout(() => resolve({ cookieBannerDismissed: false, consentHandled: false, loadMoreClicked: 0, overlaysDismissed: 0 }), 3500)),
+                    ]);
+                }
+                catch {
+                    // Never block extraction
+                }
+                throwIfAborted();
+            }
+            // Execute page actions if provided
+            if (actions && actions.length > 0) {
+                const { executeActions } = await import('./actions.js');
+                const actionScreenshot = await executeActions(page, actions);
+                if (actionScreenshot) {
+                    screenshotBuffer = actionScreenshot;
+                }
+                throwIfAborted();
+            }
+            // If the navigation returned a binary document (PDF/DOCX), grab the raw body.
+            if (isBinaryDoc) {
+                const buffer = await response.body();
+                throwIfAborted();
+                // Capture screenshot if requested (and not already captured by actions)
+                if (screenshot && !screenshotBuffer) {
+                    screenshotBuffer = await page.screenshot({
+                        fullPage: screenshotFullPage,
+                        type: 'png',
+                    });
+                }
+                return {
+                    html: '',
+                    finalUrl,
+                    buffer,
+                    contentType,
+                    statusCode: response.status(),
+                };
+            }
+            const html = await page.content();
+            throwIfAborted();
+            return {
+                html,
+                finalUrl,
+                contentType,
+                statusCode: response?.status(),
+                autoInteractResult,
+            };
+        })();
+        let operationTimeout;
+        const timeoutPromise = new Promise((_, reject) => {
+            operationTimeout = setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+        });
+        const fetchData = await Promise.race([fetchPromise, timeoutPromise]);
+        if (operationTimeout) {
+            clearTimeout(operationTimeout);
+        }
+        const { html, finalUrl } = fetchData;
+        const fetchBuffer = 'buffer' in fetchData ? fetchData.buffer : undefined;
+        const fetchContentType = 'contentType' in fetchData ? fetchData.contentType : undefined;
+        const fetchStatusCode = 'statusCode' in fetchData ? fetchData.statusCode : undefined;
+        const fetchAutoInteract = 'autoInteractResult' in fetchData ? fetchData.autoInteractResult : undefined;
+        const isBinaryDoc = !!fetchBuffer;
+        // SECURITY: Limit HTML size (skip for binary documents where html is empty)
+        if (!isBinaryDoc) {
+            if (html.length > 10 * 1024 * 1024) { // 10MB limit
+                throw new WebPeelError('Response too large (max 10MB)');
+            }
+            if (!html || html.length < 100) {
+                throw new BlockedError('Empty or suspiciously small response from browser.');
+            }
+            // Run challenge detection on browser-fetched HTML (covers both regular and stealth modes)
+            // Note: skip empty-shell type — that's a rendering quality issue (SPA needs more JS time),
+            // not a bot challenge. The caller's escalation logic handles empty-shell separately.
+            const browserChallengeResult = detectChallenge(html, fetchStatusCode);
+            if (browserChallengeResult.isChallenge && browserChallengeResult.type !== 'empty-shell') {
+                throw new BlockedError(`Challenge page detected (${browserChallengeResult.type || 'unknown'}, confidence: ${browserChallengeResult.confidence.toFixed(2)}). ` +
+                    `Site requires human verification. Try a different approach or use a CAPTCHA solving service.`);
+            }
+        }
+        // Capture screenshot if requested (and not already captured by actions or document handler)
+        if (screenshot && !screenshotBuffer) {
+            screenshotBuffer = await page.screenshot({
+                fullPage: screenshotFullPage,
+                type: 'png'
+            });
+        }
+        // If keepPageOpen, return page/browser for caller to use (e.g., branding extraction)
+        if (keepPageOpen && page) {
+            return {
+                html,
+                buffer: fetchBuffer,
+                url: finalUrl,
+                statusCode: fetchStatusCode,
+                contentType: fetchContentType,
+                screenshot: screenshotBuffer,
+                page,
+                // Use ownedBrowser for proxy case, otherwise the shared browser
+                browser: ownedBrowser ?? browser,
+                ...(fetchAutoInteract !== undefined ? { autoInteract: fetchAutoInteract } : {}),
+            };
+        }
+        return {
+            html,
+            buffer: fetchBuffer,
+            url: finalUrl,
+            statusCode: fetchStatusCode,
+            contentType: fetchContentType,
+            screenshot: screenshotBuffer,
+            ...(fetchAutoInteract !== undefined ? { autoInteract: fetchAutoInteract } : {}),
+        };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
+            throw error;
+        }
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes('Timeout')) {
+            throw new TimeoutError(`Browser navigation timed out`);
+        }
+        throw new NetworkError(`Browser fetch failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        if (signal && abortHandler) {
+            signal.removeEventListener('abort', abortHandler);
+        }
+        // CRITICAL: Always release/close page and decrement counter (unless keepPageOpen and no error)
+        if (page && !keepPageOpen) {
+            if (usingPooledPage) {
+                await recyclePooledPage(page);
+            }
+            else if (ownedContext) {
+                // Close the owned context (also closes the page)
+                await ownedContext.close().catch(() => { });
+            }
+            else if (!usingProfileBrowser) {
+                // Profile browser pages are NOT closed — the profile browser stays alive
+                // so that the next fetch in the same process reuses the session.
+                await page.close().catch(() => { });
+            }
+        }
+        // Close the dedicated proxy browser if one was launched (not when keeping page open)
+        if (ownedBrowser && !keepPageOpen) {
+            await ownedBrowser.close().catch(() => { });
+        }
+        activePagesCount--;
+    }
+}
+// ── browserScreenshot ─────────────────────────────────────────────────────────
+/**
+ * Capture a screenshot of a URL using headless Chromium via Playwright.
+ */
+export async function browserScreenshot(url, options = {}) {
+    // SECURITY: Validate URL to prevent SSRF
+    validateUrl(url);
+    const { fullPage = false, width, height, format = 'png', quality, waitMs = 0, timeoutMs = 30000, userAgent, headers, cookies, stealth = false, actions, scrollThrough = false, selector, } = options;
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    // Basic validation
+    if (waitMs < 0 || waitMs > 60000) {
+        throw new WebPeelError('Wait time must be between 0 and 60000ms');
+    }
+    if (timeoutMs < 1000 || timeoutMs > 120000) {
+        throw new WebPeelError('Timeout must be between 1000 and 120000ms');
+    }
+    if (width !== undefined && (!Number.isFinite(width) || width < 100 || width > 5000)) {
+        throw new WebPeelError('Width must be between 100 and 5000');
+    }
+    if (height !== undefined && (!Number.isFinite(height) || height < 100 || height > 5000)) {
+        throw new WebPeelError('Height must be between 100 and 5000');
+    }
+    if (format !== 'png' && format !== 'jpeg') {
+        throw new WebPeelError('Format must be png or jpeg');
+    }
+    if (format === 'jpeg' && quality !== undefined) {
+        if (!Number.isFinite(quality) || quality < 1 || quality > 100) {
+            throw new WebPeelError('JPEG quality must be between 1 and 100');
+        }
+    }
+    // SECURITY: Validate custom headers if provided
+    if (headers) {
+        for (const [key, value] of Object.entries(headers)) {
+            if (key.toLowerCase() === 'host') {
+                throw new WebPeelError('Custom Host header is not allowed');
+            }
+            if (typeof value !== 'string' || value.length > 500) {
+                throw new WebPeelError('Invalid header value');
+            }
+        }
+    }
+    // SECURITY: Limit concurrent browser pages with timeout
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000;
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    let usingPooledPage = false;
+    try {
+        const browser = stealth ? await getStealthBrowser() : await getBrowser();
+        const shouldUsePagePool = !stealth && !userAgent;
+        if (shouldUsePagePool) {
+            page = takePooledPage();
+            usingPooledPage = !!page;
+            if (usingPooledPage && getPooledPagesCount() < PAGE_POOL_SIZE) {
+                void ensurePagePool(browser).catch(() => { });
+            }
+        }
+        if (!page) {
+            page = await browser.newPage({
+                userAgent: validatedUserAgent,
+                viewport: width || height ? {
+                    width: width || 1280,
+                    height: height || 720,
+                } : null, // Use browser window size when no explicit dimensions requested
+            });
+            await applyStealthScripts(page);
+            usingPooledPage = false;
+        }
+        else {
+            await page.setViewportSize({
+                width: width || 1280,
+                height: height || 720,
+            }).catch(() => { });
+        }
+        await page.unroute('**/*').catch(() => { });
+        const mergedHeaders = { ...(headers || {}) };
+        if (usingPooledPage) {
+            mergedHeaders['User-Agent'] = validatedUserAgent;
+        }
+        if (usingPooledPage || Object.keys(mergedHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(mergedHeaders);
+        }
+        if (cookies && cookies.length > 0) {
+            const parsedCookies = cookies.map(cookie => {
+                const [nameValue] = cookie.split(';').map(s => s.trim());
+                const [name, value] = nameValue.split('=');
+                if (!name || value === undefined) {
+                    throw new WebPeelError(`Invalid cookie format: ${cookie}`);
+                }
+                return {
+                    name: name.trim(),
+                    value: value.trim(),
+                    url,
+                };
+            });
+            await page.context().addCookies(parsedCookies);
+        }
+        // For screenshots, allow all resources
+        await page.route('**/*', (route) => route.continue());
+        let screenshotBuffer;
+        const doWork = (async () => {
+            try {
+                await page.goto(url, {
+                    waitUntil: 'domcontentloaded',
+                    timeout: timeoutMs,
+                });
+            }
+            catch (gotoError) {
+                const msg = gotoError?.message || String(gotoError);
+                if (/net::ERR_HTTP2_PROTOCOL_ERROR/i.test(msg)) {
+                    throw new BlockedError(`Site blocked the request (HTTP/2 protocol error). The site likely has anti-bot protection. Try using stealth mode or a proxy.`);
+                }
+                if (/net::ERR_CONNECTION_REFUSED/i.test(msg)) {
+                    throw new NetworkError(`Connection refused by the server at ${url}. The server may be down or blocking your IP.`);
+                }
+                if (/net::ERR_CONNECTION_RESET/i.test(msg)) {
+                    throw new BlockedError(`Connection was reset by the server. This typically indicates anti-bot protection or IP blocking. Try using stealth mode or a different IP.`);
+                }
+                if (/net::ERR_SSL/i.test(msg)) {
+                    throw new NetworkError(`SSL/TLS error connecting to site. URL: ${url}`);
+                }
+                if (/net::ERR_NAME_NOT_RESOLVED/i.test(msg)) {
+                    throw new NetworkError(`Domain not found: ${url}`);
+                }
+                if (/net::ERR_CERT/i.test(msg)) {
+                    throw new NetworkError(`SSL certificate error for ${url}`);
+                }
+                if (/NS_ERROR_NET_RESET/i.test(msg)) {
+                    throw new NetworkError(`Connection reset (Firefox). The site may be blocking automated access. URL: ${url}`);
+                }
+                if (/timeout/i.test(msg)) {
+                    throw new TimeoutError(`Page load timed out after ${timeoutMs}ms: ${url}`);
+                }
+                if (/net::ERR_/i.test(msg)) {
+                    throw new NetworkError(`Browser network error: ${msg.match(/net::ERR_\w+/i)?.[0] || msg}`);
+                }
+                throw gotoError;
+            }
+            if (waitMs > 0) {
+                await page.waitForTimeout(waitMs);
+            }
+            // Element-level screenshot (clip to a specific CSS selector)
+            if (selector) {
+                const count = await page.locator(selector).count();
+                if (count === 0)
+                    throw new WebPeelError(`Element not found: ${selector}`);
+                const element = await page.locator(selector).first();
+                const buf = await element.screenshot({
+                    type: format,
+                    ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+                });
+                return { finalUrl: page.url(), screenshotBuffer: buf };
+            }
+            // Scroll through the page to trigger IntersectionObservers, lazy loading, animations
+            if (scrollThrough) {
+                await resilientScrollThrough(page, 250);
+            }
+            if (actions && actions.length > 0) {
+                const { executeActions } = await import('./actions.js');
+                const actionScreenshot = await executeActions(page, actions, {
+                    fullPage,
+                    type: format,
+                    quality,
+                });
+                if (actionScreenshot) {
+                    screenshotBuffer = actionScreenshot;
+                }
+            }
+            const finalUrl = page.url();
+            // Capture screenshot if not captured via actions
+            if (!screenshotBuffer) {
+                screenshotBuffer = await page.screenshot({
+                    fullPage,
+                    type: format,
+                    ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+                });
+            }
+            return { finalUrl, screenshotBuffer: screenshotBuffer };
+        })();
+        let operationTimeout;
+        const timeoutPromise = new Promise((_, reject) => {
+            operationTimeout = setTimeout(() => reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`)), timeoutMs);
+        });
+        const { finalUrl, screenshotBuffer: buf } = await Promise.race([doWork, timeoutPromise]);
+        if (operationTimeout) {
+            clearTimeout(operationTimeout);
+        }
+        return { buffer: buf, finalUrl };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes('Timeout')) {
+            throw new TimeoutError('Browser screenshot timed out');
+        }
+        throw new NetworkError(`Browser screenshot failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        if (page) {
+            if (usingPooledPage) {
+                await recyclePooledPage(page);
+            }
+            else {
+                await page.close().catch(() => { });
+            }
+        }
+        activePagesCount--;
+    }
+}
+// ── browserDiff ───────────────────────────────────────────────────────────────
+/**
+ * Capture screenshots of two URLs and compute a pixel-level visual diff.
+ */
+export async function browserDiff(url1, url2, options = {}) {
+    const { width = 1280, height = 720, fullPage = false, threshold = 0.1, stealth = false, waitMs = 0, timeoutMs = 30000, } = options;
+    // Take both screenshots as PNG (required for pixelmatch)
+    const [res1, res2] = await Promise.all([
+        browserScreenshot(url1, { width, height, fullPage, format: 'png', stealth, waitMs, timeoutMs }),
+        browserScreenshot(url2, { width, height, fullPage, format: 'png', stealth, waitMs, timeoutMs }),
+    ]);
+    // Dynamically import pngjs and pixelmatch (ESM-compatible)
+    const { PNG } = await import('pngjs');
+    const pixelmatch = (await import('pixelmatch')).default;
+    const img1 = PNG.sync.read(res1.buffer);
+    const img2 = PNG.sync.read(res2.buffer);
+    // Use the larger of the two dimensions
+    const outWidth = Math.max(img1.width, img2.width);
+    const outHeight = Math.max(img1.height, img2.height);
+    // Pad images to the same size if needed
+    function padImage(img, targetW, targetH) {
+        if (img.width === targetW && img.height === targetH) {
+            return img.data;
+        }
+        const padded = Buffer.alloc(targetW * targetH * 4, 0);
+        for (let y = 0; y < img.height && y < targetH; y++) {
+            for (let x = 0; x < img.width && x < targetW; x++) {
+                const srcIdx = (y * img.width + x) * 4;
+                const dstIdx = (y * targetW + x) * 4;
+                padded[dstIdx] = img.data[srcIdx];
+                padded[dstIdx + 1] = img.data[srcIdx + 1];
+                padded[dstIdx + 2] = img.data[srcIdx + 2];
+                padded[dstIdx + 3] = img.data[srcIdx + 3];
+            }
+        }
+        return padded;
+    }
+    const data1 = padImage(img1, outWidth, outHeight);
+    const data2 = padImage(img2, outWidth, outHeight);
+    const diffData = Buffer.alloc(outWidth * outHeight * 4);
+    const diffPixels = pixelmatch(data1, data2, diffData, outWidth, outHeight, { threshold });
+    const totalPixels = outWidth * outHeight;
+    const diffPercent = totalPixels > 0 ? (diffPixels / totalPixels) * 100 : 0;
+    const diffPng = new PNG({ width: outWidth, height: outHeight });
+    diffPng.data = diffData;
+    const diffBuffer = PNG.sync.write(diffPng);
+    return {
+        diffBuffer,
+        diffPixels,
+        totalPixels,
+        diffPercent,
+        dimensions: { width: outWidth, height: outHeight },
+    };
+}
+// ── retryFetch ────────────────────────────────────────────────────────────────
+/**
+ * Retry a fetch operation with exponential backoff
+ */
+export async function retryFetch(fn, maxAttempts = 3, baseDelayMs = 1000) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error('Unknown error');
+            // Don't retry on blocked errors or timeouts
+            if (error instanceof BlockedError || error instanceof TimeoutError) {
+                throw error;
+            }
+            if (attempt < maxAttempts) {
+                const delay = baseDelayMs * Math.pow(2, attempt - 1);
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+    }
+    throw lastError || new NetworkError('Retry failed');
+}
+// ── scrollAndWait ─────────────────────────────────────────────────────────────
+/**
+ * Scroll to the bottom of the page N times, waiting for the network to
+ * settle between each scroll.  Useful for triggering lazy-loaded content
+ * (infinite scroll, deferred images, etc.).
+ *
+ * @param page   - Playwright Page instance.
+ * @param times  - Number of scroll-and-wait cycles (default: 3).
+ * @returns        The final page HTML after all scrolls complete.
+ */
+export async function scrollAndWait(page, times = 3) {
+    for (let i = 0; i < times; i++) {
+        // eslint-disable-next-line @typescript-eslint/no-implied-eval
+        await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+        // Wait for network to settle (500 ms of no new requests) or 2 s max.
+        try {
+            await page.waitForLoadState('networkidle', { timeout: 2000 });
+        }
+        catch (e) {
+            // networkidle may never fire — fall back to a flat delay.
+            log.debug('networkidle timeout, falling back to flat delay:', e instanceof Error ? e.message : e);
+            await page.waitForTimeout(1000);
+        }
+    }
+    return page.content();
+}
+// ── browserFilmstrip ──────────────────────────────────────────────────────────
+/**
+ * Capture multiple screenshots at evenly distributed scroll positions.
+ * Returns an array of Buffers (one per frame).
+ */
+export async function browserFilmstrip(url, options = {}) {
+    validateUrl(url);
+    const { frames: frameCount = 6, width, height, format = 'png', quality, waitMs = 0, timeoutMs = 30000, userAgent, headers, cookies, stealth = false, } = options;
+    // Clamp frames between 2 and 12
+    const numFrames = Math.max(2, Math.min(12, frameCount));
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000;
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    try {
+        const browser = stealth ? await getStealthBrowser() : await getBrowser();
+        page = await browser.newPage({
+            userAgent: validatedUserAgent,
+            viewport: { width: width || 1280, height: height || 720 },
+        });
+        await applyStealthScripts(page);
+        if (headers)
+            await page.setExtraHTTPHeaders(headers);
+        if (cookies && cookies.length > 0) {
+            const parsedCookies = cookies.map(cookie => {
+                const [nameValue] = cookie.split(';').map(s => s.trim());
+                const [name, value] = nameValue.split('=');
+                if (!name || value === undefined) {
+                    throw new WebPeelError(`Invalid cookie format: ${cookie}`);
+                }
+                return { name: name.trim(), value: value.trim(), url };
+            });
+            await page.context().addCookies(parsedCookies);
+        }
+        await page.route('**/*', (route) => route.continue());
+        try {
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+        }
+        catch (gotoError) {
+            const msg = gotoError?.message || String(gotoError);
+            if (/timeout/i.test(msg)) {
+                throw new TimeoutError(`Page load timed out after ${timeoutMs}ms: ${url}`);
+            }
+            if (/net::ERR_/i.test(msg)) {
+                throw new NetworkError(`Browser network error: ${msg.match(/net::ERR_\w+/i)?.[0] || msg}`);
+            }
+            throw gotoError;
+        }
+        if (waitMs > 0)
+            await page.waitForTimeout(waitMs);
+        // Wait a bit for initial animations
+        await page.waitForTimeout(800);
+        const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
+        const viewportHeight = await page.evaluate(() => window.innerHeight);
+        const capturedFrames = [];
+        // Calculate scroll positions (evenly distributed)
+        const positions = [];
+        for (let i = 0; i < numFrames; i++) {
+            positions.push(Math.round((scrollHeight - viewportHeight) * i / (numFrames - 1)));
+        }
+        for (const pos of positions) {
+            await page.evaluate((y) => window.scrollTo({ top: y, behavior: 'instant' }), pos);
+            await page.waitForTimeout(350); // Let animations settle
+            const buf = await page.screenshot({
+                type: format,
+                ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+            });
+            capturedFrames.push(buf);
+        }
+        const finalUrl = page.url();
+        return { frames: capturedFrames, finalUrl };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError) {
+            throw error;
+        }
+        if (error instanceof Error && error.message.includes('Timeout')) {
+            throw new TimeoutError('Browser filmstrip timed out');
+        }
+        throw new NetworkError(`Browser filmstrip failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        if (page)
+            await page.close().catch(() => { });
+        activePagesCount--;
+    }
+}
+// ── withBrowserPage ───────────────────────────────────────────────────────────
+/**
+ * Shared boilerplate for the 4 new screenshot functions:
+ * - Queue concurrency wait
+ * - Launch browser (stealth or normal)
+ * - Open a new page with viewport + userAgent
+ * - Apply stealth scripts
+ * - Set custom headers and cookies
+ * - Navigate to the URL (with error normalisation)
+ * - Wait optional extra time
+ * - Call `fn(page)` for the unique per-function logic
+ * - Always close the page and decrement the counter
+ *
+ * NOTE: Do NOT touch browserFetch / browserScreenshot / browserFilmstrip —
+ * they have slightly different pooling / keep-open logic.
+ */
+async function withBrowserPage(url, opts, fn) {
+    validateUrl(url);
+    const { width = 1440, height = 900, userAgent, headers, cookies, stealth = false, waitMs = 0, timeoutMs = 60000, } = opts;
+    const validatedUserAgent = userAgent ? validateUserAgent(userAgent) : getRandomUserAgent();
+    const queueStartTime = Date.now();
+    const QUEUE_TIMEOUT_MS = 30000;
+    while (activePagesCount >= MAX_CONCURRENT_PAGES) {
+        if (Date.now() - queueStartTime > QUEUE_TIMEOUT_MS) {
+            throw new TimeoutError('Browser page queue timeout - too many concurrent requests');
+        }
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    activePagesCount++;
+    let page = null;
+    try {
+        const browser = stealth ? await getStealthBrowser() : await getBrowser();
+        page = await browser.newPage({
+            userAgent: validatedUserAgent,
+            viewport: { width, height },
+        });
+        await applyStealthScripts(page);
+        if (headers)
+            await page.setExtraHTTPHeaders(headers);
+        if (cookies && cookies.length > 0) {
+            const parsedCookies = cookies.map(cookie => {
+                const [nameValue] = cookie.split(';').map((s) => s.trim());
+                const [name, value] = nameValue.split('=');
+                if (!name || value === undefined)
+                    throw new WebPeelError(`Invalid cookie format: ${cookie}`);
+                return { name: name.trim(), value: value.trim(), url };
+            });
+            await page.context().addCookies(parsedCookies);
+        }
+        try {
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+        }
+        catch (gotoError) {
+            const msg = gotoError?.message || String(gotoError);
+            if (/timeout/i.test(msg))
+                throw new TimeoutError(`Page load timed out after ${timeoutMs}ms: ${url}`);
+            if (/net::ERR_/i.test(msg))
+                throw new NetworkError(`Browser network error: ${msg.match(/net::ERR_\w+/i)?.[0] || msg}`);
+            throw gotoError;
+        }
+        if (waitMs > 0)
+            await page.waitForTimeout(waitMs);
+        const result = await fn(page);
+        const finalUrl = page.url();
+        return { result, finalUrl };
+    }
+    catch (error) {
+        if (error instanceof BlockedError || error instanceof WebPeelError || error instanceof TimeoutError)
+            throw error;
+        if (error instanceof Error && error.message.includes('Timeout'))
+            throw new TimeoutError('Browser operation timed out');
+        throw new NetworkError(`Browser operation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+    finally {
+        if (page)
+            await page.close().catch(() => { });
+        activePagesCount--;
+    }
+}
+// ── browserAudit ──────────────────────────────────────────────────────────────
+/**
+ * Section-aware audit screenshots.
+ * Finds all elements matching a CSS selector and captures a viewport screenshot
+ * scrolled to each one.  Returns one image buffer per matching element.
+ */
+export async function browserAudit(url, options = {}) {
+    const { width = 1440, height = 900, format = 'jpeg', quality = 80, selector = 'section', waitMs = 0, timeoutMs = 60000, userAgent, headers, cookies, stealth = false, scrollThrough = false, } = options;
+    const { result: frames, finalUrl } = await withBrowserPage(url, { width, height, userAgent, headers, cookies, stealth, waitMs, timeoutMs }, async (page) => {
+        // Scroll through to trigger lazy content
+        if (scrollThrough) {
+            await resilientScrollThrough(page, 200);
+        }
+        const elements = await page.evaluate((sel) => {
+            const nodes = Array.from(document.querySelectorAll(sel));
+            return nodes.map(el => {
+                const rect = el.getBoundingClientRect();
+                const scrollY = window.scrollY || document.documentElement.scrollTop;
+                return {
+                    tag: el.tagName.toLowerCase(),
+                    id: el.id || '',
+                    className: el.className || '',
+                    top: rect.top + scrollY,
+                    height: rect.height,
+                };
+            });
+        }, selector);
+        const capturedFrames = [];
+        for (let i = 0; i < elements.length; i++) {
+            const el = elements[i];
+            await page.evaluate((y) => window.scrollTo({ top: y, behavior: 'instant' }), el.top);
+            await page.waitForTimeout(200);
+            const buf = await page.screenshot({
+                type: format,
+                ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+            });
+            capturedFrames.push({ index: i, ...el, buffer: buf });
+        }
+        return capturedFrames;
+    });
+    return { frames, finalUrl };
+}
+// ── browserAnimationCapture ───────────────────────────────────────────────────
+/**
+ * Capture N viewport screenshots at fixed intervals to record CSS animation states.
+ */
+export async function browserAnimationCapture(url, options = {}) {
+    const { frames: frameCount = 6, intervalMs = 500, scrollTo, selector, width = 1440, height = 900, format = 'jpeg', quality = 80, waitMs = 0, timeoutMs = 60000, userAgent, headers, cookies, stealth = false, } = options;
+    const numFrames = Math.max(1, Math.min(30, frameCount));
+    const { result: frames, finalUrl } = await withBrowserPage(url, { width, height, userAgent, headers, cookies, stealth, waitMs, timeoutMs }, async (page) => {
+        // Position the viewport
+        if (selector) {
+            await page.evaluate((sel) => {
+                const el = document.querySelector(sel);
+                if (el)
+                    el.scrollIntoView({ behavior: 'instant', block: 'start' });
+            }, selector);
+            await page.waitForTimeout(300);
+        }
+        else if (typeof scrollTo === 'number') {
+            await page.evaluate((y) => window.scrollTo({ top: y, behavior: 'instant' }), scrollTo);
+            await page.waitForTimeout(300);
+        }
+        const capturedFrames = [];
+        const startTime = Date.now();
+        for (let i = 0; i < numFrames; i++) {
+            const buf = await page.screenshot({
+                type: format,
+                ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+            });
+            capturedFrames.push({ index: i, timestampMs: Date.now() - startTime, buffer: buf });
+            if (i < numFrames - 1) {
+                await page.waitForTimeout(intervalMs);
+            }
+        }
+        return capturedFrames;
+    });
+    return { frames, finalUrl };
+}
+// ── browserViewports ──────────────────────────────────────────────────────────
+/**
+ * Capture screenshots at multiple viewport widths in a single browser session.
+ * Resizes the viewport between each capture.
+ */
+export async function browserViewports(url, options) {
+    const { viewports, fullPage = false, format = 'jpeg', quality = 80, waitMs = 0, timeoutMs = 90000, userAgent, headers, cookies, stealth = false, scrollThrough = false, } = options;
+    if (!viewports || viewports.length === 0) {
+        throw new WebPeelError('At least one viewport is required');
+    }
+    // Use first viewport dimensions for initial page setup
+    const firstVp = viewports[0];
+    const { result: frames, finalUrl } = await withBrowserPage(url, { width: firstVp.width, height: firstVp.height, userAgent, headers, cookies, stealth, waitMs, timeoutMs }, async (page) => {
+        const capturedFrames = [];
+        for (const vp of viewports) {
+            const label = vp.label || `${vp.width}x${vp.height}`;
+            // Resize viewport
+            await page.setViewportSize({ width: vp.width, height: vp.height });
+            await page.waitForTimeout(500); // Wait for reflow
+            if (scrollThrough) {
+                await resilientScrollThrough(page, 150);
+            }
+            const buf = await page.screenshot({
+                fullPage,
+                type: format,
+                ...(format === 'jpeg' && typeof quality === 'number' ? { quality } : {}),
+            });
+            capturedFrames.push({ width: vp.width, height: vp.height, label, buffer: buf });
+        }
+        return capturedFrames;
+    });
+    return { frames, finalUrl };
+}
+/**
+ * Extract computed CSS values and validate against design rules.
+ * Returns structured JSON instead of pixel images.
+ */
+export async function browserDesignAudit(url, options = {}) {
+    const { rules = {}, selector = 'body', width = 1440, height = 900, waitMs = 0, timeoutMs = 60000, userAgent, headers, cookies, stealth = false, } = options;
+    const spacingGrid = rules.spacingGrid ?? 2; // 2px base grid (accepts any even value; catches odd-pixel errors)
+    const minTouchTarget = rules.minTouchTarget ?? 44;
+    const minContrast = rules.minContrast ?? 4.5;
+    const { result: auditData, finalUrl } = await withBrowserPage(url, { width, height, userAgent, headers, cookies, stealth, waitMs, timeoutMs }, async (page) => {
+        // Run design audit inside the browser
+        return page.evaluate((params) => {
+            const { sel, spacingGrid, minTouchTarget, minContrast } = params;
+            // --- Helpers ---
+            function parsePixels(val) {
+                const n = parseFloat(val);
+                return isNaN(n) ? 0 : n;
+            }
+            function parseRgb(color) {
+                const m = color.match(/rgba?\(([0-9]+),\s*([0-9]+),\s*([0-9]+)/);
+                if (!m)
+                    return null;
+                return [parseInt(m[1]), parseInt(m[2]), parseInt(m[3])];
+            }
+            function parseRgba(color) {
+                const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/);
+                if (!m)
+                    return null;
+                return [parseInt(m[1]), parseInt(m[2]), parseInt(m[3]), m[4] !== undefined ? parseFloat(m[4]) : 1];
+            }
+            function getEffectiveBackground(el) {
+                let current = el;
+                while (current && current !== document.documentElement) {
+                    const style = window.getComputedStyle(current);
+                    const bg = style.backgroundColor;
+                    const parsed = parseRgba(bg);
+                    if (parsed && parsed[3] > 0.5) {
+                        return [parsed[0], parsed[1], parsed[2]];
+                    }
+                    current = current.parentElement;
+                }
+                // Check html element
+                const htmlStyle = window.getComputedStyle(document.documentElement);
+                const htmlBg = parseRgba(htmlStyle.backgroundColor);
+                if (htmlBg && htmlBg[3] > 0.5) {
+                    return [htmlBg[0], htmlBg[1], htmlBg[2]];
+                }
+                // Check body element
+                const bodyStyle = window.getComputedStyle(document.body);
+                const bodyBg = parseRgba(bodyStyle.backgroundColor);
+                if (bodyBg && bodyBg[3] > 0.5) {
+                    return [bodyBg[0], bodyBg[1], bodyBg[2]];
+                }
+                // Check color-scheme CSS property or meta tag
+                const colorScheme = htmlStyle.colorScheme ||
+                    document.querySelector('meta[name="color-scheme"]')?.getAttribute('content') || '';
+                if (colorScheme.includes('dark')) {
+                    return [0, 0, 0]; // Dark scheme default
+                }
+                // Ultimate fallback: white (standard web default)
+                return [255, 255, 255];
+            }
+            function hasBackdropFilter(el) {
+                let current = el;
+                while (current) {
+                    const style = window.getComputedStyle(current);
+                    const bf = style.backdropFilter;
+                    if (bf && bf !== 'none' && bf !== '')
+                        return true;
+                    current = current.parentElement;
+                }
+                return false;
+            }
+            function detectPageColorScheme() {
+                const htmlStyle = window.getComputedStyle(document.documentElement);
+                const htmlBg = parseRgba(htmlStyle.backgroundColor);
+                if (htmlBg && htmlBg[3] > 0.5) {
+                    const lum = luminance(htmlBg[0], htmlBg[1], htmlBg[2]);
+                    return lum < 0.18 ? 'dark' : 'light';
+                }
+                const bodyStyle = window.getComputedStyle(document.body);
+                const bodyBg = parseRgba(bodyStyle.backgroundColor);
+                if (bodyBg && bodyBg[3] > 0.5) {
+                    const lum = luminance(bodyBg[0], bodyBg[1], bodyBg[2]);
+                    return lum < 0.18 ? 'dark' : 'light';
+                }
+                const colorScheme = htmlStyle.colorScheme ||
+                    document.querySelector('meta[name="color-scheme"]')?.getAttribute('content') || '';
+                if (colorScheme.includes('dark'))
+                    return 'dark';
+                if (colorScheme.includes('light'))
+                    return 'light';
+                return 'unknown';
+            }
+            function luminance(r, g, b) {
+                const [rs, gs, bs] = [r, g, b].map(c => {
+                    const s = c / 255;
+                    return s <= 0.03928 ? s / 12.92 : Math.pow((s + 0.055) / 1.055, 2.4);
+                });
+                return 0.2126 * rs + 0.7152 * gs + 0.0722 * bs;
+            }
+            function contrastRatio(c1, c2) {
+                const l1 = luminance(...c1);
+                const l2 = luminance(...c2);
+                const lighter = Math.max(l1, l2);
+                const darker = Math.min(l1, l2);
+                return (lighter + 0.05) / (darker + 0.05);
+            }
+            function elementLabel(el) {
+                const id = el.id ? `#${el.id}` : '';
+                const cls = el.className && typeof el.className === 'string'
+                    ? '.' + el.className.trim().split(/\s+/).slice(0, 2).join('.')
+                    : '';
+                return `${el.tagName.toLowerCase()}${id}${cls}`;
+            }
+            function nearestMultiple(val, grid) {
+                if (grid <= 0)
+                    return val;
+                return Math.round(val / grid) * grid;
+            }
+            const root = document.querySelector(sel) || document.body;
+            const allElements = Array.from(root.querySelectorAll('*'));
+            const spacingViolations = [];
+            const touchTargetViolations = [];
+            const contrastViolations = [];
+            const fontSizesSet = new Set();
+            const lineHeightsSet = new Set();
+            const letterSpacingsSet = new Set();
+            const spacingValuesSet = new Set();
+            const interactiveTags = new Set(['a', 'button', 'input', 'select', 'textarea', 'label']);
+            for (const el of allElements) {
+                const style = window.getComputedStyle(el);
+                const rect = el.getBoundingClientRect();
+                // Skip invisible elements
+                if (rect.width === 0 && rect.height === 0)
+                    continue;
+                const label = elementLabel(el);
+                // Spacing
+                const spacingProps = ['marginTop', 'marginRight', 'marginBottom', 'marginLeft',
+                    'paddingTop', 'paddingRight', 'paddingBottom', 'paddingLeft', 'gap', 'rowGap', 'columnGap'];
+                for (const prop of spacingProps) {
+                    const raw = style[prop];
+                    if (!raw || raw === 'normal' || raw === 'auto')
+                        continue;
+                    const px = parsePixels(raw);
+                    if (px <= 0)
+                        continue;
+                    spacingValuesSet.add(px);
+                    if (spacingGrid > 0 && Math.round(px) % spacingGrid !== 0) {
+                        spacingViolations.push({
+                            element: label,
+                            property: prop,
+                            value: Math.round(px),
+                            nearestGridValue: nearestMultiple(px, spacingGrid),
+                        });
+                    }
+                }
+                // Typography
+                const fs = style.fontSize;
+                const lh = style.lineHeight;
+                const ls = style.letterSpacing;
+                if (fs)
+                    fontSizesSet.add(fs);
+                if (lh && lh !== 'normal')
+                    lineHeightsSet.add(lh);
+                if (ls && ls !== 'normal')
+                    letterSpacingsSet.add(ls);
+                // Touch targets
+                const tag = el.tagName.toLowerCase();
+                if (interactiveTags.has(tag)) {
+                    const w = rect.width;
+                    const h = rect.height;
+                    if (w > 0 && h > 0 && (w < minTouchTarget || h < minTouchTarget)) {
+                        touchTargetViolations.push({ element: label, width: Math.round(w), height: Math.round(h), minRequired: minTouchTarget });
+                    }
+                }
+                // Contrast — Walk up DOM tree to find effective opaque background
+                const textColor = style.color;
+                if (textColor) {
+                    const fg = parseRgb(textColor);
+                    if (fg) {
+                        if (hasBackdropFilter(el)) {
+                            // Background can't be determined from CSS alone — mark as unresolvable
+                            // and exclude from scoring (bgResolved: false)
+                            const text = el.textContent?.trim() || '';
+                            if (text.length > 0 && text.length < 200) {
+                                contrastViolations.push({
+                                    element: label,
+                                    textColor,
+                                    bgColor: 'unknown (backdrop-filter)',
+                                    ratio: 0,
+                                    required: minContrast,
+                                    bgResolved: false,
+                                });
+                            }
+                        }
+                        else {
+                            const effectiveBg = getEffectiveBackground(el);
+                            // bgResolved: true — background was successfully determined via DOM traversal
+                            const ratio = contrastRatio(fg, effectiveBg);
+                            if (ratio > 1.05 && ratio < minContrast) {
+                                // Only flag elements with visible text content
+                                const text = el.textContent?.trim() || '';
+                                if (text.length > 0 && text.length < 200) {
+                                    contrastViolations.push({
+                                        element: label,
+                                        textColor,
+                                        bgColor: `rgb(${effectiveBg.join(',')})`,
+                                        ratio: Math.round(ratio * 100) / 100,
+                                        required: minContrast,
+                                        bgResolved: true,
+                                    });
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            const spacingScale = Array.from(spacingValuesSet).sort((a, b) => a - b).map(v => Math.round(v));
+            // ── WCAG Accessibility Audit ──────────────────────────────────────
+            const a11yViolations = [];
+            const headingStructure = [];
+            // 1. Images without alt text
+            const images = root.querySelectorAll('img');
+            for (const img of Array.from(images)) {
+                if (!img.getAttribute('alt') && !img.getAttribute('aria-label') && !img.getAttribute('role')?.includes('presentation')) {
+                    a11yViolations.push({ type: 'missing-alt', element: elementLabel(img), details: `src: ${(img.getAttribute('src') || '').slice(0, 80)}` });
+                }
+            }
+            // 2. Form inputs without labels
+            const inputs = root.querySelectorAll('input, select, textarea');
+            for (const input of Array.from(inputs)) {
+                const id = input.getAttribute('id');
+                const hasLabel = id && document.querySelector(`label[for="${id}"]`);
+                const hasAria = input.getAttribute('aria-label') || input.getAttribute('aria-labelledby');
+                const hasTitle = input.getAttribute('title');
+                if (!hasLabel && !hasAria && !hasTitle && input.getAttribute('type') !== 'hidden') {
+                    a11yViolations.push({ type: 'missing-label', element: elementLabel(input), details: `type: ${input.getAttribute('type') || 'text'}` });
+                }
+            }
+            // 3. Heading hierarchy
+            const headings = root.querySelectorAll('h1, h2, h3, h4, h5, h6');
+            let prevLevel = 0;
+            for (const h of Array.from(headings)) {
+                const level = parseInt(h.tagName[1]);
+                headingStructure.push(h.tagName.toLowerCase());
+                if (prevLevel > 0 && level > prevLevel + 1) {
+                    a11yViolations.push({ type: 'heading-skip', element: elementLabel(h), details: `Jumped from h${prevLevel} to h${level}` });
+                }
+                prevLevel = level;
+            }
+            // 4. Empty links
+            const links = root.querySelectorAll('a');
+            for (const link of Array.from(links)) {
+                const text = (link.textContent || '').trim();
+                const aria = link.getAttribute('aria-label');
+                const title = link.getAttribute('title');
+                const hasImg = link.querySelector('img[alt]');
+                if (!text && !aria && !title && !hasImg) {
+                    a11yViolations.push({ type: 'empty-link', element: elementLabel(link), details: `href: ${(link.getAttribute('href') || '').slice(0, 60)}` });
+                }
+            }
+            // 5. Empty buttons
+            const buttons = root.querySelectorAll('button');
+            for (const btn of Array.from(buttons)) {
+                const text = (btn.textContent || '').trim();
+                const aria = btn.getAttribute('aria-label');
+                if (!text && !aria) {
+                    a11yViolations.push({ type: 'empty-button', element: elementLabel(btn), details: '' });
+                }
+            }
+            return {
+                colorScheme: detectPageColorScheme(),
+                spacingViolations: spacingViolations.slice(0, 50),
+                touchTargetViolations: touchTargetViolations.slice(0, 50),
+                contrastViolations: contrastViolations.slice(0, 50),
+                typography: {
+                    fontSizes: Array.from(fontSizesSet).slice(0, 20),
+                    lineHeights: Array.from(lineHeightsSet).slice(0, 20),
+                    letterSpacings: Array.from(letterSpacingsSet).slice(0, 20),
+                },
+                spacingScale: [...new Set(spacingScale)].slice(0, 30),
+                accessibilityViolations: a11yViolations.slice(0, 50),
+                headingStructure,
+            };
+        }, { sel: selector, spacingGrid, minTouchTarget, minContrast });
+    });
+    // Weighted scoring: contrast failures are most serious (accessibility),
+    // touch target issues affect usability, spacing is cosmetic, a11y is significant.
+    // Only count contrast violations where we could resolve the background (bgResolved: true).
+    // Violations with unresolvable backgrounds (backdrop-filter etc.) are excluded from scoring.
+    const resolvedContrastViolations = auditData.contrastViolations.filter(v => v.bgResolved !== false);
+    const unresolvedContrastViolations = auditData.contrastViolations.filter(v => v.bgResolved === false);
+    const contrastPenalty = Math.min(40, resolvedContrastViolations.length * 5); // cap at 40pts
+    const touchPenalty = Math.min(30, auditData.touchTargetViolations.length * 3); // cap at 30pts
+    const spacingPenalty = Math.min(20, auditData.spacingViolations.length * 1);
+    const a11yPenalty = Math.min(30, auditData.accessibilityViolations.length * 4);
+    // Bonus for zero violations in a category (up to 5 pts total)
+    let bonus = 0;
+    if (resolvedContrastViolations.length === 0)
+        bonus += 2;
+    if (auditData.touchTargetViolations.length === 0)
+        bonus += 1;
+    if (auditData.accessibilityViolations.length === 0)
+        bonus += 2;
+    const totalPenalty = contrastPenalty + touchPenalty + spacingPenalty + a11yPenalty;
+    const score = Math.min(100, Math.max(0, Math.round(100 - totalPenalty + bonus)));
+    const parts = [];
+    if (auditData.spacingViolations.length > 0)
+        parts.push(`${auditData.spacingViolations.length} spacing violation(s)`);
+    if (auditData.touchTargetViolations.length > 0)
+        parts.push(`${auditData.touchTargetViolations.length} touch target violation(s)`);
+    if (resolvedContrastViolations.length > 0)
+        parts.push(`${resolvedContrastViolations.length} contrast violation(s)`);
+    if (unresolvedContrastViolations.length > 0)
+        parts.push(`${unresolvedContrastViolations.length} unresolvable contrast check(s)`);
+    if (auditData.accessibilityViolations.length > 0)
+        parts.push(`${auditData.accessibilityViolations.length} accessibility violation(s)`);
+    const summary = parts.length === 0
+        ? 'No design violations found.'
+        : `Found: ${parts.join(', ')}.`;
+    const audit = { score, summary, ...auditData };
+    return { audit, finalUrl };
+}
+// ── browserDesignAnalysis ──────────────────────────────────────────────────────
+/**
+ * Extract structured visual design intelligence from a URL using a browser.
+ * Returns a DesignAnalysis object with effects, palette, layout, type scale,
+ * and quality signals.
+ */
+export async function browserDesignAnalysis(url, options = {}) {
+    const { width = 1440, height = 900, waitMs = 0, timeoutMs = 60000, userAgent, headers, cookies, stealth = false, } = options;
+    const { extractDesignAnalysis } = await import('./design-analysis.js');
+    const { result: analysis, finalUrl } = await withBrowserPage(url, { width, height, userAgent, headers, cookies, stealth, waitMs, timeoutMs }, async (page) => {
+        return extractDesignAnalysis(page);
+    });
+    return { analysis, finalUrl };
+}