jumpy-lion 0.1.1 → 0.1.2

package/dist/page.js

@@ -1,8 +1,115 @@
 import { EventEmitter } from 'events';
+import { load as cheerioLoad } from 'cheerio';
+import CDP from 'chrome-remote-interface';
+import { log } from 'crawlee';
 import { PAGE_CLOSED, PAGE_CREATED } from './events.js';
+// Constants for timing and delays
+const PRESS_RELEASE_DELAY_MS = 50; // ms between mouse press and release
+const SCROLL_ANIMATION_DELAY_MS = 200; // ms to wait for scroll animations
+const OPTION_LOAD_DELAY_MS = 300; // ms to wait for new options to load in dropdowns
+const DROPDOWN_OPEN_DELAY_MS = 300; // ms to wait for dropdown to open
+const HUMAN_DELAY_MS = 200; // ms delay for human-like interactions
+const SCROLL_COMPLETE_DELAY_MS = 300; // ms to wait for scroll to complete
+// Constants for stability and positioning
+const DEFAULT_TIMEOUT_MS = 30000; // Default timeout for operations
+const POSITION_STABILITY_TIMEOUT_MS = 2000; // Timeout for position stabilization
+const POSITION_CHECK_INTERVAL_MS = 100; // Interval for position checks
+const POSITION_STABILITY_THRESHOLD = 3; // Number of consecutive stable checks required
+const POSITION_TOLERANCE_PX = 1; // Maximum pixel difference to consider "stable"
+const VIEWPORT_TOLERANCE_PX = 50; // Tolerance for viewport positioning
+// Constants for polling and retries
+const POLLING_INTERVAL_MS = 100; // Default polling interval
+/**
+ * Creates a proxy that wraps CdpPage methods with reconnection logic
+
+ */
+// @ts-expect-error not used now
+function createReconnectingPageProxy(page) {
+    const MAX_RETRIES = 3;
+    const INITIAL_BACKOFF = 1000;
+    async function executeWithRetries(operation, methodName) {
+        let lastError = null;
+        for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+            try {
+                // Wait for any ongoing reconnection
+                while (page.isReconnecting) {
+                    await new Promise(resolve => setTimeout(resolve, 100));
+                }
+                // @ts-expect-error Accessing internal WebSocket
+                const ws = page._rawClient._ws;
+                // Check connection state before method call
+                if (ws && (ws.readyState !== ws.OPEN || ws.readyState === ws.CLOSING)) {
+                    const backoff = INITIAL_BACKOFF * 2 ** attempt;
+                    log.info(`WebSocket not connected before ${methodName}, attempting to reconnect... (attempt ${attempt + 1}/${MAX_RETRIES})`);
+                    try {
+                        await page.reconnectSession();
+                        await new Promise(resolve => setTimeout(resolve, backoff));
+                    }
+                    catch (reconnectError) {
+                        const error = reconnectError instanceof Error ? reconnectError : new Error(String(reconnectError));
+                        log.error('Reconnection failed:', { error });
+                        continue;
+                    }
+                }
+                return await operation();
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (error instanceof Error && error.message?.includes('WebSocket is not open')) {
+                    const backoff = INITIAL_BACKOFF * 2 ** attempt;
+                    log.info(`WebSocket error during ${methodName}, attempting to reconnect... (attempt ${attempt + 1}/${MAX_RETRIES})`);
+                    try {
+                        await page.reconnectSession();
+                        await new Promise(resolve => setTimeout(resolve, backoff));
+                        continue;
+                    }
+                    catch (reconnectError) {
+                        const reconnectErr = reconnectError instanceof Error ? reconnectError : new Error(String(reconnectError));
+                        log.error('Reconnection failed:', { error: reconnectErr });
+                    }
+                }
+                throw error;
+            }
+        }
+        throw lastError || new Error(`Failed to execute ${methodName} after ${MAX_RETRIES} attempts`);
+    }
+    return new Proxy(page, {
+        get(target, property) {
+            const original = target[property];
+            const propertyName = String(property);
+            // Special handling for close method
+            if (propertyName === 'close') {
+                return original;
+            }
+            // Only proxy public methods that return promises
+            if (typeof original === 'function' &&
+                !propertyName.startsWith('_') &&
+                !propertyName.startsWith('on') &&
+                propertyName !== 'emit' &&
+                propertyName !== 'addListener' &&
+                propertyName !== 'removeListener') {
+                return async (...args) => {
+                    // @ts-expect-error Accessing private property
+                    if (target.isClosed) {
+                        log.debug(`Operation '${propertyName}' skipped - page is closed`);
+                        return;
+                    }
+                    return executeWithRetries(() => original.apply(target, args), propertyName);
+                };
+            }
+            return original;
+        },
+    });
+}
 export default class CdpPage extends EventEmitter {
     constructor(client) {
         super();
+        Object.defineProperty(this, "_rawClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "client", {
             enumerable: true,
             configurable: true,
@@ -15,30 +122,143 @@ export default class CdpPage extends EventEmitter {
             writable: true,
             value: null
         });
+        Object.defineProperty(this, "isReconnecting", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "enabledDomains", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+        Object.defineProperty(this, "connectionMonitor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isClosed", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this._rawClient = client;
         this.client = client;
         this.emit(PAGE_CREATED);
+        // this._attachConnectionMonitor();
+    }
+    _attachConnectionMonitor() {
+        if (this.connectionMonitor) {
+            clearInterval(this.connectionMonitor);
+        }
+        // @ts-expect-error Accessing internal WebSocket
+        const ws = this._rawClient._ws;
+        if (!ws)
+            return;
+        ws.on('error', (error) => {
+            log.error('WebSocket error occurred:', { error });
+        });
+        // Only monitor connection health
+        this.connectionMonitor = setInterval(() => {
+            if (ws.readyState === ws.OPEN) {
+                ws.ping(() => { });
+            }
+            else if (!this.isReconnecting && ws.readyState !== ws.CONNECTING) {
+                log.debug('Connection health check failed, clearing monitor');
+                if (this.connectionMonitor)
+                    clearInterval(this.connectionMonitor);
+            }
+        }, 5000);
+        this.once(PAGE_CLOSED, () => {
+            if (this.connectionMonitor)
+                clearInterval(this.connectionMonitor);
+        });
     }
     // @TODO: The enabling of API can cause blocking.
     // We need to discover this more and add options to the crawler.
     static async create(client) {
         const page = new CdpPage(client);
         await page.initialize();
+        // const proxiedPage = createReconnectingPageProxy(page);
+        log.debug('Created proxied page instance');
         return page;
     }
     async initialize() {
-        // const { Page, DOM, Network, Runtime } = this.client;
-        // await Promise.all([Page.enable(), DOM.enable(), Network.enable(), Runtime.enable()]);
+        // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        // @TODO: NEVER ENABLE DOMAINS. THIS CAUSES BLOCKING. ALWAYS ENABLE DOMAINS ONLY IF NEEDED
+        // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+        // const { Page, DOM, Network } = this.client;
+        // Enable essential domains by default with proper error handling
+        // await Promise.all([
+        //     Page.enable().catch(e => log.error('Failed to enable Page:', e)),
+        //     DOM.enable().catch(e => log.error('Failed to enable DOM:', e)),
+        //     Network.enable().catch(e => log.error('Failed to enable Network:', e)),
+        // ]);
+    }
+    /**
+     * Lazily enable Page domain only when needed for script injection.
+     * This avoids enabling Page upfront which is a CDP detection vector.
+     * Once enabled, subsequent calls are no-ops.
+     */
+    async ensurePageEnabled() {
+        if (this.enabledDomains.has('Page')) {
+            return; // Already enabled
+        }
+        try {
+            await this.client.Page.enable();
+            this.enabledDomains.add('Page');
+            log.debug('Page domain enabled lazily for script injection');
+        }
+        catch (error) {
+            log.error('Failed to enable Page domain:', { error });
+            throw error;
+        }
+    }
+    /**
+     * Lazily enable Network domain only when needed for request interception/monitoring.
+     * This avoids enabling Network upfront which is a CDP detection vector.
+     * Once enabled, subsequent calls are no-ops.
+     */
+    async ensureNetworkEnabled() {
+        if (this.enabledDomains.has('Network')) {
+            return; // Already enabled
+        }
+        await this.client.Network.enable();
+        this.enabledDomains.add('Network');
+        log.debug('Network domain enabled (lazy)');
     }
     async url() {
         if (!this.loadedUrl) {
-            return await this.evaluate("window.location.href");
+            return await this._getCurrentUrlWithoutRuntime();
         }
         return this.loadedUrl;
     }
+    /**
+     * Get current URL without activating the Runtime domain.
+     * Uses Page.getNavigationHistory which only requires the Page domain.
+     */
+    async _getCurrentUrlWithoutRuntime() {
+        try {
+            await this.ensurePageEnabled();
+            const { currentIndex, entries } = await this.client.Page.getNavigationHistory();
+            if (entries && entries.length > 0 && currentIndex >= 0) {
+                return entries[currentIndex].url;
+            }
+        }
+        catch {
+            // Fallback: if Page.getNavigationHistory fails, try evaluate
+            log.debug('getNavigationHistory failed, falling back to evaluate');
+        }
+        return await this.evaluate('window.location.href');
+    }
     // Navigation method
     async goto(url, options) {
-        const { Page, Runtime } = this.client;
-        const timeout = options?.timeout ?? 30000;
+        const { Page } = this.client;
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
         await Page.navigate({ url });
         const waitUntil = options?.waitUntil ?? 'load';
         let waitEvent;
@@ -56,12 +276,41 @@ export default class CdpPage extends EventEmitter {
             waitEvent,
             new Promise((_, reject) => setTimeout(() => reject(new Error('Navigation timeout')), timeout)),
         ]);
-        const { result } = await Runtime.evaluate({ expression: 'window.location.href' });
-        this.loadedUrl = result.value;
+        // Use Page.getNavigationHistory instead of Runtime.evaluate to avoid
+        // activating the Runtime domain (which is detection vector #1)
+        this.loadedUrl = await this._getCurrentUrlWithoutRuntime();
+    }
+    /**
+     * Reloads the current page.
+     * @param options - Options for waiting and timeout, similar to goto.
+     */
+    async reload(options) {
+        const { Page } = this.client;
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
+        const waitUntil = options?.waitUntil ?? 'load';
+        await Page.reload({});
+        let waitEvent;
+        switch (waitUntil) {
+            case 'load':
+                waitEvent = new Promise((resolve) => Page.loadEventFired(() => resolve()));
+                break;
+            case 'domcontentloaded':
+                waitEvent = new Promise((resolve) => Page.domContentEventFired(() => resolve()));
+                break;
+            default:
+                throw new Error(`Unsupported waitUntil event: ${waitUntil}`);
+        }
+        await Promise.race([
+            waitEvent,
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Reload timeout')), timeout)),
+        ]);
+        // Use Page.getNavigationHistory instead of Runtime.evaluate
+        this.loadedUrl = await this._getCurrentUrlWithoutRuntime();
     }
     // Evaluate method
     // eslint-disable-next-line @typescript-eslint/ban-types
     async evaluate(pageFunction, ...args) {
+        await this._enableDomain('Runtime');
         const { Runtime } = this.client;
         let expression;
         if (typeof pageFunction === 'function') {
@@ -86,19 +335,349 @@ export default class CdpPage extends EventEmitter {
         return result.value;
     }
     // Click method
-    async click(selector) {
-        await this.waitForSelector(selector);
+    async click(selector, options) {
+        await Promise.all([
+            this._enableDomain('DOM'),
+            this._enableDomain('Input'),
+        ]);
         const { DOM, Input } = this.client;
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
+        const force = options?.force ?? false;
+        const delay = options?.delay ?? 0;
+        const button = options?.button ?? 'left';
+        const clickCount = options?.clickCount ?? 1;
+        // Wait for element to exist
+        await this.waitForSelector(selector, { timeout });
+        // Get element details and check visibility
+        const elementInfo = await this._getElementInfo(selector);
+        if (!elementInfo) {
+            throw new Error(`Element not found: ${selector}`);
+        }
+        // Check if element is visible (unless force is true)
+        if (!force) {
+            const isVisible = await this._isElementVisible(selector);
+            if (!isVisible) {
+                throw new Error(`Element is not visible: ${selector}. Use force: true to bypass the visibility check and attempt interaction anyway. Warning: Interacting with non-visible elements may lead to unexpected behavior or errors.`);
+            }
+        }
+        // Check if element is clickable (unless force is true)
+        if (!force) {
+            const isClickable = await this._isElementClickable(selector);
+            if (!isClickable) {
+                throw new Error(`Element is not clickable: ${selector}. Use force: true to bypass this check.`);
+            }
+        }
+        // Scroll element into view if needed
+        await this._scrollElementIntoView(selector);
+        // Wait for scroll animations to complete by polling for position stabilization
+        // Wait for the element's position to stabilize after scrolling.
+        // This is necessary because scroll animations or layout changes may cause the element's position
+        // to shift for a short period. If we attempt to click before the position is stable, the click
+        // may be sent to the wrong location, potentially missing the target element or causing unreliable
+        // automation. Polling for position stabilization ensures the click is performed accurately.
+        await this._waitForElementPositionToStabilize(selector, POSITION_STABILITY_TIMEOUT_MS, POSITION_CHECK_INTERVAL_MS);
+        // Get updated element position after scrolling
         const { nodeId } = await this.getNodeId(selector);
-        const { model } = await DOM.getBoxModel({ nodeId });
+        // For hidden elements, we need to handle the case where box model can't be computed
+        let model;
+        try {
+            const boxModelResult = await DOM.getBoxModel({ nodeId });
+            model = boxModelResult.model;
+        }
+        catch (error) {
+            if (force) {
+                // If force is true, try to get a default position
+                const fallbackElementInfo = await this._getElementInfo(selector);
+                if (fallbackElementInfo) {
+                    const x = fallbackElementInfo.boundingBox.x + fallbackElementInfo.boundingBox.width / 2;
+                    const y = fallbackElementInfo.boundingBox.y + fallbackElementInfo.boundingBox.height / 2;
+                    await this._performClick(Input, x, y, button, clickCount, delay);
+                    return;
+                }
+            }
+            throw new Error(`Could not get box model for node: ${nodeId}. Element might be hidden or not rendered.`);
+        }
         if (!model) {
             throw new Error(`Could not get box model for node: ${nodeId}`);
         }
+        // Calculate click coordinates (center of the element)
         const x = (model.content[0] + model.content[4]) / 2;
         const y = (model.content[1] + model.content[5]) / 2;
+        // Verify element is still in viewport after scrolling (with some tolerance)
+        if (!force) {
+            const isInViewport = await this._isElementInViewport(selector);
+            if (!isInViewport) {
+                // Give a bit more time for scroll animations to complete
+                await new Promise(resolve => setTimeout(resolve, HUMAN_DELAY_MS));
+                const isInViewportAfterDelay = await this._isElementInViewport(selector);
+                if (!isInViewportAfterDelay) {
+                    throw new Error(`Element is not in viewport after scrolling: ${selector}`);
+                }
+            }
+        }
+        // Perform the click with human-like interaction
+        await this._performClick(Input, x, y, button, clickCount, delay);
+    }
+    /**
+     * Performs a human-like click with proper mouse events.
+     * Note: screenX/screenY compensation is handled browser-side via fingerprint
+     * overrides that set realistic window.screenX/screenY values, which Chrome
+     * uses when constructing MouseEvent from CDP dispatch.
+     */
+    async _performClick(Input, x, y, button, clickCount, delay) {
+        // Move mouse cursor to target coordinates (x, y)
         await Input.dispatchMouseEvent({ type: 'mouseMoved', x, y });
-        await Input.dispatchMouseEvent({ type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
-        await Input.dispatchMouseEvent({ type: 'mouseReleased', x, y, button: 'left', clickCount: 1 });
+        // Small delay to simulate human behavior
+        if (delay > 0) {
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+        // Press mouse button
+        await Input.dispatchMouseEvent({
+            type: 'mousePressed',
+            x,
+            y,
+            button,
+            clickCount,
+        });
+        // Small delay between press and release
+        await new Promise(resolve => setTimeout(resolve, PRESS_RELEASE_DELAY_MS));
+        // Release mouse button
+        await Input.dispatchMouseEvent({
+            type: 'mouseReleased',
+            x,
+            y,
+            button,
+            clickCount,
+        });
+    }
+    /**
+     * Gets detailed information about an element
+     */
+    async _getElementInfo(selector) {
+        return await this.evaluate((sel) => {
+            const element = document.querySelector(sel);
+            if (!element)
+                return null;
+            const rect = element.getBoundingClientRect();
+            const style = window.getComputedStyle(element);
+            // Check if element is visible
+            const isVisible = style.display !== 'none' &&
+                style.visibility !== 'hidden' &&
+                rect.width > 0 &&
+                rect.height > 0;
+            // Check if element is clickable
+            const isClickable = !element.hasAttribute('disabled') &&
+                !element.classList.contains('disabled') &&
+                style.pointerEvents !== 'none' &&
+                rect.width > 0 &&
+                rect.height > 0;
+            return {
+                nodeId: 0, // Will be set by CDP
+                boundingBox: {
+                    x: rect.left,
+                    y: rect.top,
+                    width: rect.width,
+                    height: rect.height,
+                },
+                isVisible,
+                isClickable,
+            };
+        }, selector);
+    }
+    /**
+     * Enhanced visibility check that considers more factors
+     */
+    async _isElementVisible(selector) {
+        return await this.evaluate((sel) => {
+            const element = document.querySelector(sel);
+            if (!element)
+                return false;
+            const rect = element.getBoundingClientRect();
+            const style = window.getComputedStyle(element);
+            // Basic visibility checks
+            if (style.display === 'none' || style.visibility === 'hidden') {
+                return false;
+            }
+            // Check if element has dimensions
+            if (rect.width === 0 || rect.height === 0) {
+                return false;
+            }
+            // Check if element is not transparent
+            if (parseFloat(style.opacity) === 0) {
+                return false;
+            }
+            // Check if element is not clipped by overflow
+            const parent = element.parentElement;
+            if (parent) {
+                const parentStyle = window.getComputedStyle(parent);
+                if (parentStyle.overflow === 'hidden') {
+                    const parentRect = parent.getBoundingClientRect();
+                    if (rect.left >= parentRect.right ||
+                        rect.right <= parentRect.left ||
+                        rect.top >= parentRect.bottom ||
+                        rect.bottom <= parentRect.top) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        }, selector);
+    }
+    /**
+     * Enhanced clickable check that considers more factors
+     */
+    async _isElementClickable(selector) {
+        return await this.evaluate((sel) => {
+            const element = document.querySelector(sel);
+            if (!element)
+                return false;
+            const rect = element.getBoundingClientRect();
+            const style = window.getComputedStyle(element);
+            // Check if element is disabled
+            if (element.hasAttribute('disabled') || element.classList.contains('disabled')) {
+                return false;
+            }
+            // Check if pointer events are disabled
+            if (style.pointerEvents === 'none') {
+                return false;
+            }
+            // Check if element has dimensions
+            if (rect.width === 0 || rect.height === 0) {
+                return false;
+            }
+            // Check if element is not transparent
+            if (parseFloat(style.opacity) === 0) {
+                return false;
+            }
+            // Check if element is not read-only (for form elements)
+            if (element.hasAttribute('readonly')) {
+                return false;
+            }
+            return true;
+        }, selector);
+    }
+    /**
+     * Enhanced scroll into view with better positioning
+     */
+    async _scrollElementIntoView(selector) {
+        await this.evaluate((sel, tolerance) => {
+            const element = document.querySelector(sel);
+            if (!element)
+                return;
+            const rect = element.getBoundingClientRect();
+            const viewportHeight = window.innerHeight;
+            const viewportWidth = window.innerWidth;
+            // Check if element is already in viewport (with tolerance)
+            const isInViewport = rect.top >= -tolerance &&
+                rect.left >= -tolerance &&
+                rect.bottom <= viewportHeight + tolerance &&
+                rect.right <= viewportWidth + tolerance;
+            if (!isInViewport) {
+                // Scroll element into view with better positioning
+                element.scrollIntoView({
+                    behavior: 'smooth',
+                    block: 'center',
+                    inline: 'center',
+                });
+            }
+        }, selector, VIEWPORT_TOLERANCE_PX);
+    }
+    /**
+     * Waits for an element's position to stabilize by polling its position over time.
+     * This is useful for waiting for scroll animations or dynamic layout changes to complete.
+     *
+     * @param selector - The CSS selector for the element to monitor
+     * @param timeout - Maximum time to wait in milliseconds (default: 2000)
+     * @param checkInterval - How often to check position in milliseconds (default: 100)
+     * @param stabilityThreshold - Number of consecutive stable checks required (default: 3)
+     * @param tolerance - Maximum pixel difference to consider "stable" (default: 1)
+     */
+    async _waitForElementPositionToStabilize(selector, timeout = POSITION_STABILITY_TIMEOUT_MS, checkInterval = POSITION_CHECK_INTERVAL_MS, stabilityThreshold = POSITION_STABILITY_THRESHOLD, tolerance = POSITION_TOLERANCE_PX) {
+        const startTime = Date.now();
+        let stableCount = 0;
+        let lastPosition = null;
+        while (Date.now() - startTime < timeout) {
+            try {
+                const currentPosition = await this.evaluate((sel) => {
+                    const element = document.querySelector(sel);
+                    if (!element)
+                        return null;
+                    const rect = element.getBoundingClientRect();
+                    return {
+                        x: Math.round(rect.left),
+                        y: Math.round(rect.top),
+                        width: Math.round(rect.width),
+                        height: Math.round(rect.height),
+                    };
+                }, selector);
+                if (!currentPosition) {
+                    // Element not found, reset stability counter
+                    stableCount = 0;
+                    lastPosition = null;
+                }
+                else if (!lastPosition) {
+                    // First position reading
+                    lastPosition = currentPosition;
+                    stableCount = 1;
+                }
+                else {
+                    // Check if position is stable (within tolerance)
+                    const deltaX = Math.abs(currentPosition.x - lastPosition.x);
+                    const deltaY = Math.abs(currentPosition.y - lastPosition.y);
+                    const deltaWidth = Math.abs(currentPosition.width - lastPosition.width);
+                    const deltaHeight = Math.abs(currentPosition.height - lastPosition.height);
+                    const isStable = deltaX <= tolerance &&
+                        deltaY <= tolerance &&
+                        deltaWidth <= tolerance &&
+                        deltaHeight <= tolerance;
+                    if (isStable) {
+                        stableCount++;
+                        if (stableCount >= stabilityThreshold) {
+                            // Position has been stable for required number of checks
+                            return;
+                        }
+                    }
+                    else {
+                        // Position changed, reset counter
+                        stableCount = 1;
+                    }
+                    lastPosition = currentPosition;
+                }
+            }
+            catch (error) {
+                // Error reading position, reset stability counter
+                stableCount = 0;
+                lastPosition = null;
+            }
+            // Wait before next check
+            await new Promise(resolve => setTimeout(resolve, checkInterval));
+        }
+        // If we reach here, we timed out - this is acceptable as some elements may never stabilize
+        // If we reach here, we timed out. This means the element's position did not stabilize within the allotted time.
+        // Proceeding may cause interaction failures if the element is still moving or changing position.
+        // Callers should be aware that stabilization was not achieved and handle this case appropriately.
+    }
+    // Public wrapper for tests and external use
+    async waitForElementPositionToStabilize(selector, timeout = POSITION_STABILITY_TIMEOUT_MS, checkInterval = POSITION_CHECK_INTERVAL_MS, stabilityThreshold = POSITION_STABILITY_THRESHOLD, tolerance = POSITION_TOLERANCE_PX) {
+        return this._waitForElementPositionToStabilize(selector, timeout, checkInterval, stabilityThreshold, tolerance);
+    }
+    /**
+     * Check if element is within the viewport (with some tolerance)
+     */
+    async _isElementInViewport(selector) {
+        return await this.evaluate((sel, tolerance) => {
+            const element = document.querySelector(sel);
+            if (!element)
+                return false;
+            const rect = element.getBoundingClientRect();
+            const viewportHeight = window.innerHeight;
+            const viewportWidth = window.innerWidth;
+            // Add some tolerance for scroll positioning
+            return rect.top >= -tolerance &&
+                rect.left >= -tolerance &&
+                rect.bottom <= viewportHeight + tolerance &&
+                rect.right <= viewportWidth + tolerance;
+        }, selector, VIEWPORT_TOLERANCE_PX);
     }
     // Type method
     async type(selector, text, options) {
@@ -113,6 +692,17 @@ export default class CdpPage extends EventEmitter {
             }
         }
     }
+    /**
+     * Deletes (clears) the value of an input field specified by selector.
+     * @param selector - The CSS selector for the input element.
+     */
+    async deleteInput(selector) {
+        await this.evaluate((sel) => {
+            const input = document.querySelector(sel);
+            if (input)
+                input.value = '';
+        }, selector);
+    }
     // Set viewport
     async setViewport(viewport) {
         const { Emulation } = this.client;
@@ -135,23 +725,22 @@ export default class CdpPage extends EventEmitter {
     }
     // Wait for selector
     async waitForSelector(selector, options) {
-        const timeout = options?.timeout ?? 30000;
-        const pollingInterval = 100;
-        const maxAttempts = Math.ceil(timeout / pollingInterval);
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
+        const maxAttempts = Math.ceil(timeout / POLLING_INTERVAL_MS);
         let attempts = 0;
         while (attempts < maxAttempts) {
             const exists = await this.elementExists(selector);
             if (exists) {
                 return;
             }
-            await new Promise((resolve) => setTimeout(resolve, pollingInterval));
+            await new Promise((resolve) => setTimeout(resolve, POLLING_INTERVAL_MS));
             attempts++;
         }
         throw new Error(`Timeout waiting for selector: ${selector}`);
     }
     // Wait for response
     async waitForResponse(urlPart, statusCode = 200, timeout = 180000) {
-        await this.client.Network.enable(); // Ensure Network monitoring is enabled
+        await this.ensureNetworkEnabled(); // Lazy enable Network only when needed
         return new Promise((resolve, reject) => {
             let timer;
             const onResponseReceived = async ({ requestId, response }) => {
@@ -168,7 +757,7 @@ export default class CdpPage extends EventEmitter {
                         // Retry logic with exponential backoff
                         for (let attempt = 1; attempt <= 3; attempt++) {
                             try {
-                                await this.client.Network.enable();
+                                await this.ensureNetworkEnabled();
                                 await new Promise((resolveTimeout) => setTimeout(resolveTimeout, 1000 * 2 ** attempt)); // Exponential backoff
                                 const { body } = await this.client.Network.getResponseBody({ requestId });
                                 resolve({
@@ -192,14 +781,15 @@ export default class CdpPage extends EventEmitter {
             }, timeout);
         });
     }
-    // Screenshot method
     async screenshot(options) {
         const { Page } = this.client;
+        const format = options?.format ?? 'png';
+        // Use captureBeyondViewport for full page screenshots to avoid changing viewport
+        const screenshotOptions = { format };
         if (options?.fullPage) {
-            const { contentSize } = await Page.getLayoutMetrics();
-            await this.setViewport({ width: contentSize.width, height: contentSize.height });
+            screenshotOptions.captureBeyondViewport = true;
         }
-        const { data } = await Page.captureScreenshot({ format: 'png' });
+        const { data } = await Page.captureScreenshot(screenshotOptions);
         const buffer = Buffer.from(data, 'base64');
         if (options?.path) {
             const fs = await import('fs');
@@ -211,9 +801,19 @@ export default class CdpPage extends EventEmitter {
     async content() {
         return await this.evaluate(() => document.documentElement.outerHTML);
     }
+    /* Converts the current page content to a Cheerio instance.
+     * @returns { Promise<cheerio.CheerioAPI> } A Promise that resolves to a Cheerio API instance.
+      */
+    async toCheerio() {
+        const html = await this.content();
+        const loaded = cheerioLoad(html);
+        return loaded;
+        ;
+    }
     // Close method
     async close() {
         const { Page } = this.client;
+        this.isClosed = true;
         await Page.close();
         this.emit(PAGE_CLOSED);
     }
@@ -246,10 +846,51 @@ export default class CdpPage extends EventEmitter {
             }
         }, selector);
     }
+    // Add proper Fetch Metadata headers to avoid CDP detection
+    async setupFetchMetadataHeaders() {
+        const { Network } = this.client;
+        await this._enableDomain('Network');
+        await Network.setRequestInterception({ patterns: [{ urlPattern: '*' }] });
+        Network.on('requestIntercepted', async (params) => {
+            const { request, interceptionId } = params;
+            const url = new URL(request.url);
+            // Add proper fetch metadata headers
+            const headers = { ...request.headers };
+            // Set sec-fetch-* headers based on request type and destination
+            if (request.method === 'GET' && !headers['sec-fetch-dest']) {
+                if (url.pathname.endsWith('.js')) {
+                    headers['sec-fetch-dest'] = 'script';
+                    headers['sec-fetch-mode'] = 'no-cors';
+                    headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+                }
+                else if (url.pathname.endsWith('.css')) {
+                    headers['sec-fetch-dest'] = 'style';
+                    headers['sec-fetch-mode'] = 'no-cors';
+                    headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+                }
+                else if (url.pathname.match(/\.(png|jpg|jpeg|gif|webp|svg)$/i)) {
+                    headers['sec-fetch-dest'] = 'image';
+                    headers['sec-fetch-mode'] = 'no-cors';
+                    headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+                }
+                else if (url.pathname === '/' || !url.pathname.includes('.')) {
+                    headers['sec-fetch-dest'] = 'document';
+                    headers['sec-fetch-mode'] = 'navigate';
+                    headers['sec-fetch-site'] = 'none';
+                    headers['sec-fetch-user'] = '?1';
+                }
+            }
+            // Continue the request with modified headers
+            await Network.continueInterceptedRequest({
+                interceptionId,
+                headers,
+            });
+        });
+    }
     // Block requests matching URL patterns
     async blockRequests(urlPatterns) {
         const { Network } = this.client;
-        await Network.enable();
+        await this.ensureNetworkEnabled(); // Lazy enable Network only when needed
         await Network.setRequestInterception({ patterns: [{ urlPattern: '*' }] });
         Network.on('requestIntercepted', async (params) => {
             const shouldBlock = urlPatterns.some((pattern) => {
@@ -319,30 +960,374 @@ export default class CdpPage extends EventEmitter {
         }, selector, pageFunction.toString(), ...args);
     }
     /**
- * Adds a script to be evaluated on every new document initialization.
- *
- * @param {string | Function} script - The JavaScript code or function to inject.
- * @returns {Promise<void>} - Resolves when the script has been added.
- */
-    async addInitScript(script) {
-        const { Page } = this.client;
-        let scriptText;
-        if (typeof script === 'function') {
-            // Convert the function to a string and extract its body
-            const fnStr = script.toString();
-            const bodyMatch = fnStr.match(/function\s*\(\)\s*{([\s\S]*)}/);
-            if (!bodyMatch) {
-                throw new Error('Invalid function format for init script.');
+     * Enables a CDP domain only if it hasn't been enabled yet
+     * @private
+     */
+    async _enableDomain(domain) {
+        if (this.enabledDomains.has(domain))
+            return;
+        const domainApi = this.client[domain];
+        if (domainApi?.enable) {
+            await domainApi.enable();
+            this.enabledDomains.add(domain);
+            log.debug(`Enabled ${domain} domain`);
+        }
+    }
+    async reconnectSession(retries = 3) {
+        if (this.isReconnecting) {
+            log.debug('Reconnection skipped - already in progress');
+            return;
+        }
+        if (this.isClosed) {
+            log.debug('Reconnection skipped - page is closed');
+            return;
+        }
+        let attempt = 0;
+        this.isReconnecting = true;
+        try {
+            while (attempt < retries) {
+                try {
+                    // Wait with exponential backoff before retrying
+                    if (attempt > 0) {
+                        const backoffMs = Math.min(1000 * (2 ** attempt), 10000);
+                        log.info(`Waiting ${backoffMs}ms before reconnection attempt ${attempt + 1}/${retries}`);
+                        await new Promise(resolve => setTimeout(resolve, backoffMs));
+                    }
+                    // Create a fresh CDP connection
+                    const newSession = await CDP({
+                        port: this._rawClient.port,
+                        local: true,
+                    });
+                    try {
+                        // Get available targets
+                        const { targetInfos } = await newSession.Target.getTargets();
+                        const pageTarget = targetInfos.find(t => t.type === 'page' && !t.url.includes('chrome://'));
+                        if (!pageTarget) {
+                            log.debug('No valid page target found, stopping reconnection');
+                            return;
+                        }
+                        // Connect to the specific page target
+                        const pageSession = await CDP({
+                            target: pageTarget.targetId,
+                            port: this._rawClient.port,
+                            local: true,
+                        });
+                        // Re-enable only the domains that were previously enabled
+                        const enablePromises = Array.from(this.enabledDomains).map(domain => pageSession[domain].enable());
+                        await Promise.all(enablePromises);
+                        // Update references
+                        this._rawClient = pageSession;
+                        this.client = pageSession;
+                        this._attachConnectionMonitor();
+                        log.info(`Page session reconnected successfully on attempt ${attempt + 1}`);
+                        return;
+                    }
+                    catch (error) {
+                        // Clean up the session if anything fails
+                        await newSession.close().catch(() => { });
+                        throw error;
+                    }
+                }
+                catch (error) {
+                    attempt++;
+                    if (attempt === retries) {
+                        log.debug('Reconnection stopped - max retries reached');
+                        return;
+                    }
+                    log.error('Failed to reconnect page session', { error });
+                }
             }
-            scriptText = bodyMatch[1];
         }
-        else if (typeof script === 'string') {
-            scriptText = script;
+        finally {
+            this.isReconnecting = false;
+        }
+    }
+    /**
+     * Selects one or more options from a select element or dropdown.
+     *
+     * Automatically handles both regular HTML select elements and custom dropdowns:
+     * - For <select> elements: Directly selects options within the select
+     * - For custom dropdowns: Automatically detects and clicks triggers to open dropdowns,
+     *   then selects the specified options with intelligent scrolling for virtualized lists
+     *
+     * @param dropdownSelector - CSS selector for the select element or dropdown container
+     * @param optionSelector - CSS selector(s) for the option(s) to select. Can be a single selector or array
+     * @param options - Optional configuration for timeout, scrolling, visibility checks, etc.
+     *
+     * @example
+     * // Regular HTML select
+     * await page.selectOption('select#country', 'option[value="us"]');
+     *
+     * // Custom dropdown (automatically finds and clicks trigger)
+     * await page.selectOption('#dropdown-menu', '[data-value="premium"]');
+     *
+     * // Multiple selection
+     * await page.selectOption('select#languages', ['option[value="en"]', 'option[value="es"]']);
+     */
+    async selectOption(dropdownSelector, optionSelector, options) {
+        await Promise.all([
+            this._enableDomain('DOM'),
+            this._enableDomain('Input'),
+        ]);
+        // DOM and Input domains are enabled and actively used in this method
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
+        const force = options?.force ?? false;
+        const waitForOptions = options?.waitForOptions ?? true;
+        const maxScrollAttempts = options?.maxScrollAttempts ?? 10;
+        // Normalize target selectors to array
+        const targetSelectors = Array.isArray(optionSelector) ? optionSelector : [optionSelector];
+        // Wait for dropdown element to exist
+        await this.waitForSelector(dropdownSelector, { timeout });
+        // Check if this is a regular select element
+        const isSelectElement = await this.evaluate((sel) => {
+            const element = document.querySelector(sel);
+            return element?.tagName === 'SELECT';
+        }, dropdownSelector);
+        // For regular select elements, do visibility and disabled checks upfront
+        if (isSelectElement) {
+            // Check if element is visible (unless force is true)
+            if (!force) {
+                const isVisible = await this._isElementVisible(dropdownSelector);
+                if (!isVisible) {
+                    throw new Error(`Element is not visible: ${dropdownSelector}. Use force: true to bypass this check.`);
+                }
+            }
+            // Check if element is disabled (unless force is true)
+            if (!force) {
+                const isDisabled = await this.evaluate((sel) => {
+                    const element = document.querySelector(sel);
+                    return element?.hasAttribute('disabled') || element?.getAttribute('disabled') === 'true';
+                }, dropdownSelector);
+                if (isDisabled) {
+                    throw new Error(`Element is disabled: ${dropdownSelector}. Use force: true to bypass this check.`);
+                }
+            }
+            // Scroll element into view if needed
+            await this._scrollElementIntoView(dropdownSelector);
+            // Wait for scroll animations to complete
+            await new Promise(resolve => setTimeout(resolve, SCROLL_ANIMATION_DELAY_MS));
+        }
+        if (isSelectElement) {
+            // Handle regular select element
+            await this._handleRegularSelect(dropdownSelector, targetSelectors, {
+                timeout,
+                force,
+            });
         }
         else {
-            throw new Error('Script must be a string or a function');
+            // Handle custom dropdown - the dropdownSelector is the container
+            await this._handleCustomDropdown(dropdownSelector, targetSelectors, {
+                timeout,
+                force,
+                waitForOptions,
+                maxScrollAttempts,
+            });
+        }
+    }
+    /**
+     * Handles selection for regular HTML select elements
+     */
+    async _handleRegularSelect(selectSelector, targetSelectors, _options) {
+        const result = await this.evaluate((sel, targetSels) => {
+            const select = document.querySelector(sel);
+            if (!select) {
+                throw new Error(`Select element not found: ${sel}`);
+            }
+            // Check if multiple selection is allowed
+            if (!select.multiple && targetSels.length > 1) {
+                throw new Error(`Select element does not support multiple selection. Found ${targetSels.length} selectors but multiple is false.`);
+            }
+            // Clear previous selections
+            if (select.multiple) {
+                Array.from(select.options).forEach(option => {
+                    option.selected = false;
+                });
+            }
+            let foundCount = 0;
+            const foundSelectors = [];
+            // Select the options by CSS selector
+            for (const targetSelector of targetSels) {
+                let optionFound = false;
+                // Find option by CSS selector within the select element
+                const option = select.querySelector(targetSelector);
+                if (option) {
+                    option.selected = true;
+                    optionFound = true;
+                    foundSelectors.push(targetSelector);
+                }
+                if (optionFound) {
+                    foundCount++;
+                }
+            }
+            // Trigger change event
+            const event = new Event('change', { bubbles: true });
+            select.dispatchEvent(event);
+            return { foundCount, foundSelectors, totalOptions: select.options.length };
+        }, selectSelector, targetSelectors);
+        if (result.foundCount === 0) {
+            throw new Error(`No options found for selectors: ${targetSelectors.join(', ')}. Available options: ${result.totalOptions}`);
         }
-        await Page.addScriptToEvaluateOnNewDocument({ source: scriptText });
+        if (result.foundCount < targetSelectors.length) {
+            const missingSelectors = targetSelectors.filter(s => !result.foundSelectors.includes(s));
+            throw new Error(`Some selectors not found: ${missingSelectors.join(', ')}. Found: ${result.foundSelectors.join(', ')}`);
+        }
+    }
+    /**
+     * Handles selection for custom dropdown elements.
+     * Automatically detects and clicks triggers using multiple strategies:
+     * - Common trigger patterns ([aria-haspopup], .dropdown-trigger, etc.)
+     * - Previous sibling elements
+     * - ID pattern matching (menu → trigger)
+     */
+    async _handleCustomDropdown(dropdownContainer, targetSelectors, options) {
+        const { timeout, force, waitForOptions, maxScrollAttempts, dropdownOpenDelay = DROPDOWN_OPEN_DELAY_MS } = options;
+        // For custom dropdowns, always try to find and click a trigger to open
+        // Most custom dropdowns need a trigger click to become visible
+        await this.evaluate((containerSel) => {
+            const container = document.querySelector(containerSel);
+            if (!container)
+                return false;
+            // Strategy 1: Look for sibling trigger elements with common patterns
+            const parent = container.parentElement;
+            if (parent) {
+                const triggers = parent.querySelectorAll('[role="button"], button, [data-toggle], [aria-haspopup], .dropdown-trigger, .select-trigger');
+                for (const trigger of triggers) {
+                    if (trigger !== container) {
+                        trigger.click();
+                        return true;
+                    }
+                }
+                // Strategy 2: Look for previous sibling that might be the trigger
+                const prevSibling = container.previousElementSibling;
+                if (prevSibling && prevSibling.tagName !== 'LABEL') {
+                    prevSibling.click();
+                    return true;
+                }
+            }
+            // Strategy 3: Look for element with similar ID (e.g., menu -> trigger)
+            const containerId = container.id;
+            if (containerId) {
+                const triggerPatterns = [
+                    containerId.replace('menu', 'trigger').replace('dropdown', 'trigger'),
+                    containerId.replace('-menu', '-trigger'),
+                    containerId.replace('_menu', '_trigger'),
+                    containerId + '-trigger',
+                    'trigger-' + containerId,
+                ];
+                for (const triggerId of triggerPatterns) {
+                    const trigger = document.getElementById(triggerId);
+                    if (trigger && trigger !== container) {
+                        trigger.click();
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }, dropdownContainer);
+        // Wait for dropdown to open
+        await new Promise(resolve => setTimeout(resolve, dropdownOpenDelay));
+        // Wait for dropdown to be visible
+        if (waitForOptions) {
+            await this.waitForSelector(dropdownContainer, { timeout });
+        }
+        // Now check if dropdown is visible (unless force is true)
+        if (!force) {
+            const isVisible = await this._isElementVisible(dropdownContainer);
+            if (!isVisible) {
+                throw new Error(`Dropdown container is not visible after trigger click: ${dropdownContainer}. Use force: true to bypass this check.`);
+            }
+        }
+        // Select each target selector
+        for (const targetSelector of targetSelectors) {
+            await this._selectOptionFromDropdown(dropdownContainer, targetSelector, {
+                timeout,
+                force,
+                maxScrollAttempts,
+            });
+        }
+    }
+    /**
+     * Selects a single option from a dropdown container.
+     * Handles virtualized lists with intelligent scrolling to find options that may not be initially visible.
+     */
+    async _selectOptionFromDropdown(dropdownSelector, targetSelector, dropdownOptions) {
+        const { timeout, maxScrollAttempts } = dropdownOptions;
+        const startTime = Date.now();
+        let scrollAttempts = 0;
+        let optionFound = false;
+        while (scrollAttempts < maxScrollAttempts && !optionFound && (Date.now() - startTime) < timeout) {
+            // Try to find and click the option
+            const result = await this.evaluate((dropdownSel, targetSel) => {
+                const dropdown = document.querySelector(dropdownSel);
+                if (!dropdown)
+                    return { found: false, needsScroll: false };
+                // Try to find the target element directly first
+                let targetOption = dropdown.querySelector(targetSel);
+                if (!targetOption) {
+                    // Check if we need to scroll to find more options
+                    const isScrollable = dropdown.scrollHeight > dropdown.clientHeight;
+                    const isAtBottom = dropdown.scrollTop + dropdown.clientHeight >= dropdown.scrollHeight - 1;
+                    return { found: false, needsScroll: isScrollable && !isAtBottom };
+                }
+                // Check if option is visible within the dropdown
+                const rect = targetOption.getBoundingClientRect();
+                const dropdownRect = dropdown.getBoundingClientRect();
+                const isVisible = rect.top >= dropdownRect.top &&
+                    rect.bottom <= dropdownRect.bottom &&
+                    rect.left >= dropdownRect.left &&
+                    rect.right <= dropdownRect.right;
+                if (!isVisible) {
+                    // Scroll to make the option visible
+                    targetOption.scrollIntoView({ block: 'center', behavior: 'smooth' });
+                    return { found: false, needsScroll: false, scrolled: true };
+                }
+                // Click the option
+                targetOption.click();
+                return { found: true, needsScroll: false };
+            }, dropdownSelector, targetSelector);
+            if (result.found) {
+                optionFound = true;
+                break;
+            }
+            if (result.needsScroll) {
+                // Scroll down to load more options
+                await this.evaluate((dropdownSel) => {
+                    const dropdown = document.querySelector(dropdownSel);
+                    if (dropdown) {
+                        dropdown.scrollTop += dropdown.clientHeight * 0.8;
+                    }
+                }, dropdownSelector);
+                scrollAttempts++;
+                await new Promise(resolve => setTimeout(resolve, OPTION_LOAD_DELAY_MS)); // Wait for new options to load
+            }
+            else if (result.scrolled) {
+                // Option was scrolled into view, try clicking it now
+                await new Promise(resolve => setTimeout(resolve, SCROLL_COMPLETE_DELAY_MS)); // Wait for scroll to complete
+            }
+            else {
+                // No more scrolling needed, option not found
+                break;
+            }
+        }
+        if (!optionFound) {
+            throw new Error(`Option "${targetSelector}" not found in dropdown "${dropdownSelector}"`);
+        }
+    }
+    /**
+     * Checks if the element specified by selector is visible (not display:none and not visibility:hidden).
+     * The selector should be the root item which can be hidden, otherwise this function could return a false positive.
+     * @param selector - The CSS selector for the element.
+     * @returns true if the element is visible, false otherwise.
+     */
+    async isVisible(selector) {
+        return await this.evaluate((s) => {
+            const el = document.querySelector(s);
+            if (!el) {
+                return false;
+            }
+            const style = window.getComputedStyle(el);
+            return style.display !== 'none' && style.visibility !== 'hidden';
+        }, selector);
     }
 }
 //# sourceMappingURL=page.js.map