@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/internals/enqueue-links/click-elements.js

@@ -0,0 +1,336 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clickElements = exports.isTargetRelevant = exports.clickElementsAndInterceptNavigationRequests = exports.enqueueLinksByClickingElements = void 0;
+const tslib_1 = require("tslib");
+const browser_1 = require("@crawlee/browser");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const url_1 = require("url");
+const puppeteer_request_interception_1 = require("../utils/puppeteer_request_interception");
+const STARTING_Z_INDEX = 2147400000;
+const log = log_1.default.child({ prefix: 'Puppeteer Click Elements' });
+/**
+ * The function finds elements matching a specific CSS selector in a Puppeteer page,
+ * clicks all those elements using a mouse move and a left mouse button click and intercepts
+ * all the navigation requests that are subsequently produced by the page. The intercepted
+ * requests, including their methods, headers and payloads are then enqueued to a provided
+ * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available
+ * in `href` elements, but rather navigations are triggered in click handlers.
+ * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
+ *
+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
+ * and override settings of the enqueued {@link Request} objects.
+ *
+ * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,
+ * it is recommended to only use this function as the last operation in the page.
+ *
+ * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements
+ * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.
+ *
+ * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation
+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting
+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements
+ * that you assume or know will produce a navigation. You can certainly click everything by using
+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.
+ *
+ * **Example usage**
+ *
+ * ```javascript
+ * await Actor.utils.puppeteer.enqueueLinksByClickingElements({
+ *   page,
+ *   requestQueue,
+ *   selector: 'a.product-detail',
+ *   pseudoUrls: [
+ *       'https://www.example.com/handbags/[.*]'
+ *       'https://www.example.com/purses/[.*]'
+ *   ],
+ * });
+ * ```
+ *
+ * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
+ */
+async function enqueueLinksByClickingElements(options) {
+    (0, ow_1.default)(options, ow_1.default.object.exactShape({
+        page: ow_1.default.object.hasKeys('goto', 'evaluate'),
+        requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
+        selector: ow_1.default.string,
+        clickOptions: ow_1.default.optional.object.hasKeys('clickCount', 'delay'),
+        pseudoUrls: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('purl'))),
+        globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
+        regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
+        transformRequestFunction: ow_1.default.optional.function,
+        waitForPageIdleSecs: ow_1.default.optional.number,
+        maxWaitForPageIdleSecs: ow_1.default.optional.number,
+    }));
+    const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, } = options;
+    const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
+    const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const urlPatternObjects = [];
+    if (pseudoUrls?.length) {
+        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        urlPatternObjects.push(...(0, browser_1.constructRegExpObjectsFromPseudoUrls)(pseudoUrls));
+    }
+    if (globs?.length) {
+        urlPatternObjects.push(...(0, browser_1.constructGlobObjectsFromGlobs)(globs));
+    }
+    if (regexps?.length) {
+        urlPatternObjects.push(...(0, browser_1.constructRegExpObjectsFromRegExps)(regexps));
+    }
+    const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
+        page,
+        selector,
+        waitForPageIdleMillis,
+        maxWaitForPageIdleMillis,
+        clickOptions,
+    });
+    let requestOptions = (0, browser_1.createRequestOptions)(interceptedRequests);
+    if (transformRequestFunction) {
+        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
+    }
+    const requests = (0, browser_1.createRequests)(requestOptions, urlPatternObjects);
+    return requestQueue.addRequests(requests);
+}
+exports.enqueueLinksByClickingElements = enqueueLinksByClickingElements;
+/**
+ * Clicks all elements of given page matching given selector.
+ * Catches and intercepts all initiated navigation requests and opened pages.
+ * Returns a list of all target URLs.
+ * @ignore
+ */
+async function clickElementsAndInterceptNavigationRequests(options) {
+    const { page, selector, waitForPageIdleMillis, maxWaitForPageIdleMillis, clickOptions, } = options;
+    const uniqueRequests = new Set();
+    const browser = page.browser();
+    const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);
+    const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);
+    const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);
+    await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, onInterceptedRequest);
+    browser.on("targetcreated" /* TargetCreated */, onTargetCreated);
+    page.on("framenavigated" /* FrameNavigated */, onFrameNavigated);
+    await preventHistoryNavigation(page);
+    await clickElements(page, selector, clickOptions);
+    await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });
+    await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
+    browser.off("targetcreated" /* TargetCreated */, onTargetCreated);
+    page.off("framenavigated" /* FrameNavigated */, onFrameNavigated);
+    await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, onInterceptedRequest);
+    const serializedRequests = Array.from(uniqueRequests);
+    return serializedRequests.map((r) => JSON.parse(r));
+}
+exports.clickElementsAndInterceptNavigationRequests = clickElementsAndInterceptNavigationRequests;
+/**
+ * @ignore
+ */
+function createInterceptRequestHandler(page, requests) {
+    return async function onInterceptedRequest(req) {
+        if (!isTopFrameNavigationRequest(page, req))
+            return req.continue();
+        const url = req.url();
+        requests.add(JSON.stringify({
+            url,
+            headers: req.headers(),
+            method: req.method(),
+            payload: req.postData(),
+        }));
+        if (req.redirectChain().length) {
+            await req.respond({ body: '' }); // Prevents 301/302 redirect
+        }
+        else {
+            await req.abort('aborted'); // Prevents navigation by js
+        }
+    };
+}
+/**
+ * @ignore
+ */
+function isTopFrameNavigationRequest(page, req) {
+    return req.isNavigationRequest()
+        && req.frame() === page.mainFrame();
+}
+/**
+ * @ignore
+ */
+function createTargetCreatedHandler(page, requests) {
+    return async function onTargetCreated(target) {
+        if (!isTargetRelevant(page, target))
+            return;
+        const url = target.url();
+        requests.add(JSON.stringify({ url }));
+        // We want to close the page but don't care about
+        // possible errors like target closed.
+        try {
+            const createdPage = await target.page();
+            await createdPage.close();
+        }
+        catch (err) {
+            log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
+        }
+    };
+}
+/**
+ * We're only interested in pages created by the page we're currently clicking in.
+ * There will generally be a lot of other targets being created in the browser.
+ */
+function isTargetRelevant(page, target) {
+    return target.type() === 'page'
+        && page.target() === target.opener();
+}
+exports.isTargetRelevant = isTargetRelevant;
+/**
+ * @ignore
+ */
+function createFrameNavigatedHandler(page, requests) {
+    return function onFrameNavigated(frame) {
+        if (frame !== page.mainFrame())
+            return;
+        const url = frame.url();
+        requests.add(JSON.stringify({ url }));
+    };
+}
+/**
+ * @ignore
+ */
+async function preventHistoryNavigation(page) {
+    /* istanbul ignore next */
+    return page.evaluate(() => {
+        window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangle
+        delete window.history; // Simple override does not work.
+        window.history = {
+            stateHistory: [],
+            length: 0,
+            state: {},
+            go() { },
+            back() { },
+            forward() { },
+            pushState(...args) {
+                this.stateHistory.push(args);
+            },
+            replaceState(...args) {
+                this.stateHistory.push(args);
+            },
+        };
+    });
+}
+/**
+ * Click all elements matching the given selector. To be able to do this using
+ * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
+ * so we first move them to the top of the page's stacking context and then click.
+ * We do all in series to prevent elements from hiding one another. Therefore,
+ * for large element sets, this will take considerable amount of time.
+ * @ignore
+ */
+async function clickElements(page, selector, clickOptions) {
+    const elementHandles = await page.$$(selector);
+    log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
+    let clickedElementsCount = 0;
+    let zIndex = STARTING_Z_INDEX;
+    let shouldLogWarning = true;
+    for (const handle of elementHandles) {
+        try {
+            await page.evaluate(updateElementCssToEnableMouseClick, handle, zIndex++);
+            await handle.click(clickOptions);
+            clickedElementsCount++;
+        }
+        catch (err) {
+            const e = err;
+            if (shouldLogWarning && e.stack.includes('is detached from document')) {
+                log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. `
+                    + 'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. '
+                    + 'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
+                shouldLogWarning = false;
+            }
+            log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
+        }
+    }
+    log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
+}
+exports.clickElements = clickElements;
+/* istanbul ignore next */
+/**
+ * This is an in browser function!
+ */
+function updateElementCssToEnableMouseClick(el, zIndex) {
+    el.style.visibility = 'visible';
+    el.style.display = 'block';
+    el.style.position = 'fixed';
+    el.style.zIndex = String(zIndex);
+    el.style.left = '0';
+    el.style.top = '0';
+    const boundingRect = el.getBoundingClientRect();
+    if (!boundingRect.height)
+        el.style.height = '10px';
+    if (!boundingRect.width)
+        el.style.width = '10px';
+}
+/**
+ * This function tracks whether any requests, frame navigations or targets were emitted
+ * in the past idleIntervalMillis and whenever the interval registers no activity,
+ * the function returns.
+ *
+ * It will also return when a final timeout, represented by the timeoutMillis parameter
+ * is reached, to prevent blocking on pages with constant network activity.
+ *
+ * We need this to make sure we don't finish too soon when intercepting requests triggered
+ * by clicking in the page. They often get registered by the Node.js process only some
+ * milliseconds after clicking and we would lose those requests. This is especially prevalent
+ * when there's only a single element to click.
+ * @ignore
+ */
+async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis }) {
+    return new Promise((resolve) => {
+        let timeout;
+        let maxTimeout;
+        const context = page.browserContext();
+        function newTabTracker(target) {
+            if (isTargetRelevant(page, target))
+                activityHandler();
+        }
+        function activityHandler() {
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
+                clearTimeout(maxTimeout);
+                finish();
+            }, waitForPageIdleMillis);
+        }
+        function maxTimeoutHandler() {
+            log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. `
+                + 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
+            finish();
+        }
+        function finish() {
+            page.off("request" /* Request */, activityHandler);
+            page.off("framenavigated" /* FrameNavigated */, activityHandler);
+            context.off("targetcreated" /* TargetCreated */, newTabTracker);
+            resolve();
+        }
+        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        activityHandler(); // We call this once manually in case there would be no requests at all.
+        page.on("request" /* Request */, activityHandler);
+        page.on("framenavigated" /* FrameNavigated */, activityHandler);
+        context.on("targetcreated" /* TargetCreated */, newTabTracker);
+    });
+}
+/**
+ * @ignore
+ */
+async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
+    /* eslint-disable no-shadow */
+    /* istanbul ignore next */
+    const state = await page.evaluate(() => {
+        const { stateHistory } = window.history;
+        window.history = window.__originalHistory__; // eslint-disable-line no-underscore-dangle
+        return stateHistory;
+    });
+    state.forEach((args) => {
+        try {
+            const stateUrl = args[args.length - 1];
+            const url = new url_1.URL(stateUrl, page.url()).href;
+            requests.add(JSON.stringify({ url }));
+        }
+        catch (err) {
+            log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
+        }
+    });
+}
+//# sourceMappingURL=click-elements.js.map

package/dist/internals/enqueue-links/click-elements.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":";;;;AAAA,8CAa0B;AAC1B,6DAA8B;AAE9B,oDAAoB;AAWpB,6BAA0B;AAC1B,4FAAoH;AAEpH,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AA2H/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACI,KAAK,UAAU,8BAA8B,CAAC,OAA8C;IAC/F,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,QAAQ,EAAE,YAAE,CAAC,MAAM;QACnB,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7C,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,GAC7B,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,8CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,uCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,2CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,IAAA,8BAAoB,EAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpF;IACD,MAAM,QAAQ,GAAG,IAAA,wBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACnE,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAnED,wEAmEC;AAaD;;;;;GAKG;AACI,KAAK,UAAU,2CAA2C,CAAC,OAA2D;IACzH,MAAM,EACF,IAAI,EACJ,QAAQ,EACR,qBAAqB,EACrB,wBAAwB,EACxB,YAAY,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,eAAe,GAAG,0BAA0B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,sCAAqC,eAAe,CAAC,CAAC;IAChE,IAAI,CAAC,EAAE,wCAAmC,gBAAgB,CAAC,CAAC;IAE5D,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,sCAAqC,eAAe,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,wCAAmC,gBAAgB,CAAC,CAAC;IAC7D,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAEhE,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAjCD,kGAiCC;AAED;;GAEG;AACH,SAAS,6BAA6B,CAAC,IAAU,EAAE,QAAqB;IACpE,OAAO,KAAK,UAAU,oBAAoB,CAAC,GAAG;QAC1C,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACxB,GAAG;YACH,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACpB,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE;SAC1B,CAAC,CAAC,CAAC;QAEJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE;YAC5B,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;SAChE;aAAM;YACH,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;SAC3D;IACL,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,GAAqB;IAClE,OAAO,GAAG,CAAC,mBAAmB,EAAE;WACzB,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,IAAU,EAAE,QAAqB;IACjE,OAAO,KAAK,UAAU,eAAe,CAAC,MAAM;QACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI;YACA,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,WAAY,CAAC,KAAK,EAAE,CAAC;SAC9B;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC/G;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,IAAU,EAAE,MAAc;IACvD,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM;WACxB,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC;AAHD,4CAGC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,QAAqB;IAClE,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAaD;;GAEG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAU;IAC9C,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrB,MAAgC,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,2CAA2C;QACnH,OAAQ,MAAgC,CAAC,OAAO,CAAC,CAAC,iCAAiC;QAClF,MAAgC,CAAC,OAAO,GAAG;YACxC,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAe;gBACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAe;gBAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACW,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,YAA2B;IACzF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACjC,IAAI;YACA,MAAM,IAAI,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;SAC1B;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,gBAAgB,IAAI,CAAC,CAAC,KAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE;gBACpE,GAAG,CAAC,OAAO,CAAC,4BAA4B,QAAQ,+DAA+D;sBACzG,qHAAqH;sBACrH,uHAAuH,CAAC,CAAC;gBAC/H,gBAAgB,GAAG,KAAK,CAAC;aAC5B;YACD,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SAClF;KACJ;IACD,GAAG,CAAC,KAAK,CAAC,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;AACvI,CAAC;AAvBD,sCAuBC;AAED,0BAA0B;AAC1B;;GAEG;AACH,SAAS,kCAAkC,CAAC,EAAe,EAAE,MAAc;IACvE,EAAE,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IAChC,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,MAAM,YAAY,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;IAChD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACnD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAA0B;IAC5G,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,IAAI,OAAuB,CAAC;QAC5B,IAAI,UAA0B,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtC,SAAS,aAAa,CAAC,MAAc;YACjC,IAAI,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;gBAAE,eAAe,EAAE,CAAC;QAC1D,CAAC;QAED,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,GAAG,CAAC,KAAK,CAAC,oEAAoE,wBAAwB,MAAM;kBACtG,4GAA4G,CAAC,CAAC;YACpH,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,GAAG,0BAA4B,eAAe,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,wCAAmC,eAAe,CAAC,CAAC;YAC5D,OAAO,CAAC,GAAG,sCAA4C,aAAa,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,eAAe,EAAE,CAAC,CAAC,wEAAwE;QAC3F,IAAI,CAAC,EAAE,0BAA4B,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE,wCAAmC,eAAe,CAAC,CAAC;QAC3D,OAAO,CAAC,EAAE,sCAA4C,aAAa,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAU,EAAE,QAAqB;IACxF,8BAA8B;IAC9B,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAiC,CAAC;QACjE,MAAgC,CAAC,OAAO,GAAI,MAAgC,CAAC,mBAAmB,CAAC,CAAC,2CAA2C;QAC9I,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI;YACA,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC5F;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/internals/puppeteer-crawler.d.ts

@@ -0,0 +1,149 @@
+import { BrowserCrawler, BrowserCrawlerHandleRequest, BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook } from '@crawlee/browser';
+import { PuppeteerPlugin } from '@crawlee/browser-pool';
+import { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
+import { PuppeteerLaunchContext } from './puppeteer-launcher';
+import { DirectNavigationOptions } from './utils/puppeteer_utils';
+export declare type PuppeteerController = ReturnType<PuppeteerPlugin['_createController']>;
+export declare type PuppeteerCrawlContext = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;
+export declare type PuppeteerHook = BrowserHook<PuppeteerCrawlContext, PuppeteerGoToOptions>;
+export declare type PuppeteerRequestHandlerParam = BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController>;
+export declare type PuppeteerRequestHandler = BrowserCrawlerHandleRequest<PuppeteerRequestHandlerParam>;
+export declare type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
+export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<PuppeteerCrawlContext, PuppeteerGoToOptions, {
+    browserPlugins: [PuppeteerPlugin];
+}> {
+    /**
+     * Options used by {@link launchPuppeteer} to start new Puppeteer instances.
+     */
+    launchContext?: PuppeteerLaunchContext;
+    /**
+     * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
+     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
+     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * Example:
+     * ```
+     * preNavigationHooks: [
+     *     async (crawlingContext, gotoOptions) => {
+     *         const { page } = crawlingContext;
+     *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *     },
+     * ]
+     * ```
+     */
+    preNavigationHooks?: PuppeteerHook[];
+    /**
+     * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
+     * The function accepts `crawlingContext` as the only parameter.
+     * Example:
+     * ```
+     * postNavigationHooks: [
+     *     async (crawlingContext) => {
+     *         const { page } = crawlingContext;
+     *         if (hasCaptcha(page)) {
+     *             await solveCaptcha (page);
+     *         }
+     *     },
+     * ]
+     * ```
+     */
+    postNavigationHooks?: PuppeteerHook[];
+}
+/**
+ * Provides a simple framework for parallel crawling of web pages
+ * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `PuppeteerCrawler` uses headless Chrome to download web pages and extract data,
+ * it is useful for crawling of websites that require to execute JavaScript.
+ * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * which downloads the pages using raw HTTP requests and is about 10x faster.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
+ * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.handlePageFunction} option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
+ *
+ * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ page, request }) => {
+ *         // This function is called to extract data from a single web page
+ *         // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ *         // 'request' is an instance of Request class with information about the page to load
+ *         await Actor.pushData({
+ *             title: await page.title(),
+ *             url: request.url,
+ *             succeeded: true,
+ *         })
+ *     },
+ *     handleFailedRequestFunction: async ({ request }) => {
+ *         // This function is called when the crawling of a request failed too many times
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             succeeded: false,
+ *             errors: request.errorMessages,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+export declare class PuppeteerCrawler extends BrowserCrawler<{
+    browserPlugins: [PuppeteerPlugin];
+}, LaunchOptions, PuppeteerCrawlContext> {
+    protected static optionsShape: {
+        browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        gotoFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        gotoTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+    };
+    /**
+     * All `PuppeteerCrawler` parameters are passed via an options object.
+     */
+    constructor(options: PuppeteerCrawlerOptions);
+    protected _navigationHandler(crawlingContext: PuppeteerCrawlContext, gotoOptions: DirectNavigationOptions): Promise<any>;
+}
+//# sourceMappingURL=puppeteer-crawler.d.ts.map

package/dist/internals/puppeteer-crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EACd,2BAA2B,EAC3B,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACd,MAAM,kBAAkB,CAAC;AAI1B,OAAO,EAAsB,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAqB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,uBAAuB,EAAgB,MAAM,yBAAyB,CAAC;AAEhF,oBAAY,mBAAmB,GAAG,UAAU,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAEnF,oBAAY,qBAAqB,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAEnG,oBAAY,aAAa,GAAG,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;AAErF,oBAAY,4BAA4B,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAA;AAE1G,oBAAY,uBAAuB,GAAG,2BAA2B,CAAC,4BAA4B,CAAC,CAAC;AAEhG,oBAAY,oBAAoB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,uBAAwB,SAAQ,qBAAqB,CAClE,qBAAqB,EACrB,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,CACxC;IACG;;OAEG;IACH,aAAa,CAAC,EAAE,sBAAsB,CAAC;IAEvC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,aAAa,EAAE,CAAC;IAErC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,aAAa,EAAE,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,qBAAa,gBAAiB,SAAQ,cAAc,CAAC;IAAE,cAAc,EAAE,CAAC,eAAe,CAAC,CAAA;CAAE,EAAE,aAAa,EAAE,qBAAqB,CAAC;IAC7H,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF;;OAEG;gBACS,OAAO,EAAE,uBAAuB;cAqCnB,kBAAkB,CAAC,eAAe,EAAE,qBAAqB,EAAE,WAAW,EAAE,uBAAuB;CAQ3H"}

package/dist/internals/puppeteer-crawler.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PuppeteerCrawler = void 0;
+const tslib_1 = require("tslib");
+const browser_1 = require("@crawlee/browser");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const puppeteer_launcher_1 = require("./puppeteer-launcher");
+const stealth_1 = require("./stealth");
+const puppeteer_utils_1 = require("./utils/puppeteer_utils");
+/**
+ * Provides a simple framework for parallel crawling of web pages
+ * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `PuppeteerCrawler` uses headless Chrome to download web pages and extract data,
+ * it is useful for crawling of websites that require to execute JavaScript.
+ * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
+ * which downloads the pages using raw HTTP requests and is about 10x faster.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
+ * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.handlePageFunction} option.
+ *
+ * New pages are only opened when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
+ *
+ * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ page, request }) => {
+ *         // This function is called to extract data from a single web page
+ *         // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ *         // 'request' is an instance of Request class with information about the page to load
+ *         await Actor.pushData({
+ *             title: await page.title(),
+ *             url: request.url,
+ *             succeeded: true,
+ *         })
+ *     },
+ *     handleFailedRequestFunction: async ({ request }) => {
+ *         // This function is called when the crawling of a request failed too many times
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             succeeded: false,
+ *             errors: request.errorMessages,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+class PuppeteerCrawler extends browser_1.BrowserCrawler {
+    /**
+     * All `PuppeteerCrawler` parameters are passed via an options object.
+     */
+    constructor(options) {
+        (0, ow_1.default)(options, 'PuppeteerCrawlerOptions', ow_1.default.object.exactShape(PuppeteerCrawler.optionsShape));
+        const { launchContext = {}, browserPoolOptions = {}, proxyConfiguration, ...browserCrawlerOptions } = options;
+        const { stealth = false } = launchContext;
+        if (launchContext.proxyUrl) {
+            throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
+                + 'Use PuppeteerCrawlerOptions.proxyConfiguration');
+        }
+        const puppeteerLauncher = new puppeteer_launcher_1.PuppeteerLauncher(launchContext);
+        browserPoolOptions.browserPlugins = [
+            puppeteerLauncher.createBrowserPlugin(),
+        ];
+        super({ ...browserCrawlerOptions, proxyConfiguration, browserPoolOptions });
+        if (stealth) {
+            this.browserPool.postLaunchHooks.push(async (_pageId, browserController) => {
+                // TODO: We can do this better now. It is not necessary to override the page.
+                //  we can modify the page in the postPageCreateHook
+                const { hideWebDriver, ...newStealthOptions } = puppeteerLauncher.stealthOptions;
+                (0, stealth_1.applyStealthToBrowser)(browserController.browser, newStealthOptions);
+            });
+        }
+        this.launchContext = launchContext;
+    }
+    async _navigationHandler(crawlingContext, gotoOptions) {
+        if (this.gotoFunction) {
+            this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
+            return this.gotoFunction(crawlingContext, gotoOptions);
+        }
+        return (0, puppeteer_utils_1.gotoExtended)(crawlingContext.page, crawlingContext.request, gotoOptions);
+    }
+}
+exports.PuppeteerCrawler = PuppeteerCrawler;
+Object.defineProperty(PuppeteerCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...browser_1.BrowserCrawler.optionsShape,
+        browserPoolOptions: ow_1.default.optional.object,
+        launchContext: ow_1.default.optional.object,
+    }
+});
+//# sourceMappingURL=puppeteer-crawler.js.map

package/dist/internals/puppeteer-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer-crawler.js","sourceRoot":"","sources":["../../src/internals/puppeteer-crawler.ts"],"names":[],"mappings":";;;;AAAA,8CAM0B;AAK1B,oDAAoB;AAEpB,6DAAiF;AACjF,uCAAkD;AAClD,6DAAgF;AA0DhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AACH,MAAa,gBAAiB,SAAQ,wBAA2F;IAO7H;;OAEG;IACH,YAAY,OAAgC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAwB,EAC7C,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,aAAa,CAAC;QAE1C,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QAED,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAE5E,IAAI,OAAO,EAAE;YACT,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE;gBACvE,6EAA6E;gBAC7E,oDAAoD;gBACpD,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,iBAAiB,CAAC,cAAe,CAAC;gBAClF,IAAA,+BAAqB,EAAC,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAsC,EAAE,WAAoC;QACpH,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAyB,CAAC,CAAC;SACxE;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAtDL,4CAuDC;AAtDG;;;;WAAyC;QACrC,GAAG,wBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;GAAC"}