apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/enqueue_links/click_elements.js

@@ -1,434 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.clickElements = exports.isTargetRelevant = exports.clickElementsAndInterceptNavigationRequests = exports.enqueueLinksByClickingElements = void 0;
-const tslib_1 = require("tslib");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const url_1 = require("url");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
-const puppeteer_request_interception_1 = require("../puppeteer_request_interception");
-const shared_1 = require("./shared");
-/* eslint-enable import/named,no-unused-vars,import/order */
-const STARTING_Z_INDEX = 2147400000;
-/**
- * The function finds elements matching a specific CSS selector in a Puppeteer page,
- * clicks all those elements using a mouse move and a left mouse button click and intercepts
- * all the navigation requests that are subsequently produced by the page. The intercepted
- * requests, including their methods, headers and payloads are then enqueued to a provided
- * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available
- * in `href` elements, but rather navigations are triggered in click handlers.
- * If you're looking to find URLs in `href` attributes of the page, see {@link utils#enqueueLinks}.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
- *
- * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
- * such as changing the Z-index of elements being clicked and their visibility. Therefore,
- * it is recommended to only use this function as the last operation in the page.
- *
- * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements
- * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.
- *
- * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation
- * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting
- * the scope of the clicking as much as possible by using a specific selector that targets only the elements
- * that you assume or know will produce a navigation. You can certainly click everything by using
- * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.
- *
- * **Example usage**
- *
- * ```javascript
- * await Apify.utils.puppeteer.enqueueLinksByClickingElements({
- *   page,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
- *   ],
- * });
- * ```
- * @param {object} options
- *   All `enqueueLinksByClickingElements()` parameters are passed
- *   via an options object with the following keys:
- * @param {Page} options.page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {RequestQueue} options.requestQueue
- *   A request queue to which the URLs will be enqueued.
- * @param {string} options.selector
- *   A CSS selector matching elements to be clicked on. Unlike in {@link utils#enqueueLinks}, there is no default
- *   value. This is to prevent suboptimal use of this function by using it too broadly.
- * @param {Array<(string|RegExp|Object<string, *>)>} [options.pseudoUrls]
- *   An array of {@link PseudoUrl}s matching the URLs to be enqueued,
- *   or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
- *
- *   The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.
- *   All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,
- *   which lets you specify special properties for the enqueued {@link Request} objects.
- *
- *   If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
- *   enqueues all links found on the page.
- * @param {object} [options.clickOptions]
- *   click options for use in Puppeteer's click handler
- * @param {number} [options.clickOptions.clickCount]
- *   Number of clicks to be executed. Defaults to 1
- * @param {number} [options.clickOptions.delay]
- *   Time to wait between mousedown and mouseup in milliseconds. Defaults to 0
- * @param {RequestTransform} [options.transformRequestFunction]
- *   Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
- *   to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- *   when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- *   or to dynamically update or create `userData`.
- *
- *   For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
- *   a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
- *   (POST requests).
- *
- *   **Example:**
- *   ```javascript
- *   {
- *       transformRequestFunction: (request) => {
- *           request.userData.foo = 'bar';
- *           request.useExtendedUniqueKey = true;
- *           return request;
- *       }
- *   }
- *   ```
- * @param {number} [options.waitForPageIdleSecs=1]
- *   Clicking in the page triggers various asynchronous operations that lead to new URLs being shown
- *   by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser.
- *   These events often happen only some time after the actual click. Requests typically take milliseconds
- *   while new tabs open in hundreds of milliseconds.
- *
- *   To be able to capture all those events, the `enqueueLinksByClickingElements()` function repeatedly waits
- *   for the `waitForPageIdleSecs`. By repeatedly we mean that whenever a relevant event is triggered, the timer
- *   is restarted. As long as new events keep coming, the function will not return, unless
- *   the below `maxWaitForPageIdleSecs` timeout is reached.
- *
- *   You may want to reduce this for example when you're sure that your clicks do not open new tabs,
- *   or increase when you're not getting all the expected URLs.
- * @param {number} [options.maxWaitForPageIdleSecs=5]
- *   This is the maximum period for which the function will keep tracking events, even if more events keep coming.
- *   Its purpose is to prevent a deadlock in the page by periodic events, often unrelated to the clicking itself.
- *   See `waitForPageIdleSecs` above for an explanation.
- * @return {Promise<Array<QueueOperationInfo>>}
- *   Promise that resolves to an array of {@link QueueOperationInfo} objects.
- * @memberOf puppeteer
- * @name enqueueLinksByClickingElements
- * @function
- */
-async function enqueueLinksByClickingElements(options) {
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        page: ow_1.default.object.hasKeys('goto', 'evaluate'),
-        requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
-        clickOptions: ow_1.default.optional.object.hasKeys('clickCount', 'delay'),
-        selector: ow_1.default.string,
-        pseudoUrls: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('purl'))),
-        transformRequestFunction: ow_1.default.optional.function,
-        waitForPageIdleSecs: ow_1.default.optional.number,
-        maxWaitForPageIdleSecs: ow_1.default.optional.number,
-    }));
-    const { page, requestQueue, selector, pseudoUrls, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, clickOptions, } = options;
-    const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
-    const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
-    const pseudoUrlInstances = (0, shared_1.constructPseudoUrlInstances)(pseudoUrls || []);
-    const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
-        page,
-        selector,
-        waitForPageIdleMillis,
-        maxWaitForPageIdleMillis,
-        clickOptions,
-    });
-    let requestOptions = (0, shared_1.createRequestOptions)(interceptedRequests);
-    if (transformRequestFunction) {
-        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
-    }
-    const requests = (0, shared_1.createRequests)(requestOptions, pseudoUrlInstances);
-    return (0, shared_1.addRequestsToQueueInBatches)(requests, requestQueue);
-}
-exports.enqueueLinksByClickingElements = enqueueLinksByClickingElements;
-/**
- * Clicks all elements of given page matching given selector.
- * Catches and intercepts all initiated navigation requests and opened pages.
- * Returns a list of all target URLs.
- *
- * @param {object} options
- * @param {Page} options.page
- * @param {string} options.selector
- * @param {number} [options.waitForPageIdleMillis]
- * @param {number} [options.maxWaitForPageIdleMillis]
- * @param {object} [clickOptions]
- * @param {number} [clickOptions.clickCount]
- * @param {number} [clickOptions.delay]
- * @return {Promise<Array<*>>}
- * @ignore
- */
-async function clickElementsAndInterceptNavigationRequests(options) {
-    const { page, selector, waitForPageIdleMillis, maxWaitForPageIdleMillis, clickOptions, } = options;
-    const uniqueRequests = new Set();
-    const browser = page.browser();
-    const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);
-    const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);
-    const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);
-    await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, onInterceptedRequest);
-    browser.on('targetcreated', onTargetCreated);
-    page.on('framenavigated', onFrameNavigated);
-    await preventHistoryNavigation(page);
-    await clickElements(page, selector, clickOptions);
-    await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });
-    await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
-    browser.removeListener('targetcreated', onTargetCreated);
-    page.removeListener('framenavigated', onFrameNavigated);
-    await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, onInterceptedRequest);
-    const serializedRequests = Array.from(uniqueRequests);
-    return serializedRequests.map((r) => JSON.parse(r));
-}
-exports.clickElementsAndInterceptNavigationRequests = clickElementsAndInterceptNavigationRequests;
-/**
- * @param {Page} page
- * @param {Set<*>} requests
- * @return {Function}
- * @ignore
- */
-function createInterceptRequestHandler(page, requests) {
-    return function onInterceptedRequest(req) {
-        if (!isTopFrameNavigationRequest(page, req))
-            return req.continue();
-        const url = req.url();
-        requests.add(JSON.stringify({
-            url,
-            headers: req.headers(),
-            method: req.method(),
-            payload: req.postData(),
-        }));
-        if (req.redirectChain().length) {
-            req.respond({ body: '' }); // Prevents 301/302 redirect
-        }
-        else {
-            req.abort('aborted'); // Prevents navigation by js
-        }
-    };
-}
-/**
- * @param {Page} page
- * @param {PuppeteerRequest} req
- * @return {boolean}
- * @ignore
- */
-function isTopFrameNavigationRequest(page, req) {
-    return req.isNavigationRequest()
-        && req.frame() === page.mainFrame();
-}
-/**
- * @param {Page} page
- * @param {Set<*>} requests
- * @return {Function}
- * @ignore
- */
-function createTargetCreatedHandler(page, requests) {
-    return async function onTargetCreated(target) {
-        if (!isTargetRelevant(page, target))
-            return;
-        const url = target.url();
-        requests.add(JSON.stringify({ url }));
-        // We want to close the page but don't care about
-        // possible errors like target closed.
-        try {
-            const createdPage = await target.page();
-            await createdPage.close();
-        }
-        catch (err) {
-            utils_log_1.default.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
-        }
-    };
-}
-/**
- * We're only interested in pages created by the page we're currently clicking in.
- * There will generally be a lot of other targets being created in the browser.
- * @param {Page} page
- * @param {Target} target
- * @return {boolean}
- */
-function isTargetRelevant(page, target) {
-    return target.type() === 'page'
-        && page.target() === target.opener();
-}
-exports.isTargetRelevant = isTargetRelevant;
-/**
- * @param {Page} page
- * @param {Set<*>} requests
- * @return {Function}
- * @ignore
- */
-function createFrameNavigatedHandler(page, requests) {
-    return function onFrameNavigated(frame) {
-        if (frame !== page.mainFrame())
-            return;
-        const url = frame.url();
-        requests.add(JSON.stringify({ url }));
-    };
-}
-/**
- * @param {Page} page
- * @return {Promise<*>}
- * @ignore
- */
-async function preventHistoryNavigation(page) {
-    /* istanbul ignore next */
-    return page.evaluate(() => {
-        window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangle
-        delete window.history; // Simple override does not work.
-        window.history = {
-            stateHistory: [],
-            length: 0,
-            state: {},
-            go() { },
-            back() { },
-            forward() { },
-            pushState(...args) {
-                this.stateHistory.push(args);
-            },
-            replaceState(...args) {
-                this.stateHistory.push(args);
-            },
-        };
-    });
-}
-/**
- * Click all elements matching the given selector. To be able to do this using
- * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
- * so we first move them to the top of the page's stacking context and then click.
- * We do all in series to prevent elements from hiding one another. Therefore,
- * for large element sets, this will take considerable amount of time.
- *
- * @param {Page} page
- * @param {string} selector
- * @param {object} [clickOptions]
- * @param {number} [clickOptions.clickCount]
- * @param {number} [clickOptions.delay]
- * @return {Promise<void>}
- * @ignore
- */
-async function clickElements(page, selector, clickOptions) {
-    const elementHandles = await page.$$(selector);
-    utils_log_1.default.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
-    let clickedElementsCount = 0;
-    let zIndex = STARTING_Z_INDEX;
-    let shouldLogWarning = true;
-    for (const handle of elementHandles) {
-        try {
-            await page.evaluate(updateElementCssToEnableMouseClick, handle, zIndex++);
-            await handle.click(clickOptions);
-            clickedElementsCount++;
-        }
-        catch (err) {
-            if (shouldLogWarning && err.stack.includes('is detached from document')) {
-                utils_log_1.default.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. `
-                    + 'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. '
-                    + 'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
-                shouldLogWarning = false;
-            }
-            utils_log_1.default.debug('enqueueLinksByClickingElements: Click failed.', { stack: err.stack });
-        }
-    }
-    utils_log_1.default.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
-}
-exports.clickElements = clickElements;
-/* istanbul ignore next */
-/**
- * This is an in browser function!
- * @param {Element} el
- * @param {number} zIndex
- */
-function updateElementCssToEnableMouseClick(el, zIndex) {
-    el.style.visibility = 'visible';
-    el.style.display = 'block';
-    el.style.position = 'fixed';
-    el.style.zIndex = zIndex;
-    el.style.left = 0;
-    el.style.top = 0;
-    const boundingRect = el.getBoundingClientRect();
-    if (!boundingRect.height)
-        el.style.height = '10px';
-    if (!boundingRect.width)
-        el.style.width = '10px';
-}
-/**
- * This function tracks whether any requests, frame navigations or targets were emitted
- * in the past idleIntervalMillis and whenever the interval registers no activity,
- * the function returns.
- *
- * It will also return when a final timeout, represented by the timeoutMillis parameter
- * is reached, to prevent blocking on pages with constant network activity.
- *
- * We need this to make sure we don't finish too soon when intercepting requests triggered
- * by clicking in the page. They often get registered by the Node.js process only some
- * milliseconds after clicking and we would lose those requests. This is especially prevalent
- * when there's only a single element to click.
- *
- * @param {Object} options
- * @param {Page} options.page
- * @param {number} options.waitForPageIdleMillis
- * @param {number} options.maxWaitForPageIdleMillis
- * @return {Promise<void>}
- * @ignore
- */
-async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis }) {
-    return new Promise((resolve) => {
-        let timeout;
-        let maxTimeout;
-        const context = page.browserContext();
-        function newTabTracker(target) {
-            if (isTargetRelevant(page, target))
-                activityHandler();
-        }
-        function activityHandler() {
-            clearTimeout(timeout);
-            timeout = setTimeout(() => {
-                clearTimeout(maxTimeout);
-                finish();
-            }, waitForPageIdleMillis);
-        }
-        function maxTimeoutHandler() {
-            utils_log_1.default.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. `
-                + 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
-            finish();
-        }
-        function finish() {
-            page.removeListener('request', activityHandler);
-            page.removeListener('framenavigated', activityHandler);
-            context.removeListener('targetcreated', newTabTracker);
-            resolve();
-        }
-        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
-        timeout = activityHandler(); // We call this once manually in case there would be no requests at all.
-        page.on('request', activityHandler);
-        page.on('framenavigated', activityHandler);
-        context.on('targetcreated', newTabTracker);
-    });
-}
-/**
- * @param {Page} page
- * @param {Set<*>} requests
- * @return {Promise<void>}
- * @ignore
- */
-async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
-    /* eslint-disable no-shadow */
-    /* istanbul ignore next */
-    const stateHistory = await page.evaluate(() => {
-        const { stateHistory } = window.history;
-        window.history = window.__originalHistory__; // eslint-disable-line no-underscore-dangle
-        return stateHistory;
-    });
-    stateHistory.forEach((args) => {
-        try {
-            const stateUrl = args[args.length - 1];
-            const url = new url_1.URL(stateUrl, page.url()).href;
-            requests.add(JSON.stringify({ url }));
-        }
-        catch (err) {
-            utils_log_1.default.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
-        }
-    });
-}
-//# sourceMappingURL=click_elements.js.map

package/build/enqueue_links/click_elements.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"click_elements.js","sourceRoot":"","sources":["../../src/enqueue_links/click_elements.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AACpB,6BAA0B;AAC1B,0EAA+B;AAE/B,sFAA8G;AAG9G,qCAMkB;AAClB,4DAA4D;AAE5D,MAAM,gBAAgB,GAAG,UAAU,CAAC;AAEpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0GG;AACI,KAAK,UAAU,8BAA8B,CAAC,OAAO;IACxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,QAAQ,EAAE,YAAE,CAAC,MAAM;QACnB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7F,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7C,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,EAC1B,YAAY,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,kBAAkB,GAAG,IAAA,oCAA2B,EAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IACzE,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpF;IACD,MAAM,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IACpE,OAAO,IAAA,oCAA2B,EAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;AAC/D,CAAC;AAxCD,wEAwCC;AAED;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,2CAA2C,CAAC,OAAO;IACrE,MAAM,EACF,IAAI,EACJ,QAAQ,EACR,qBAAqB,EACrB,wBAAwB,EACxB,YAAY,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,eAAe,GAAG,0BAA0B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAC7C,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IAE5C,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,OAAO,CAAC,cAAc,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IACzD,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IACxD,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAEhE,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAjCD,kGAiCC;AAED;;;;;GAKG;AACH,SAAS,6BAA6B,CAAC,IAAI,EAAE,QAAQ;IACjD,OAAO,SAAS,oBAAoB,CAAC,GAAG;QACpC,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACxB,GAAG;YACH,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACpB,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE;SAC1B,CAAC,CAAC,CAAC;QAEJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE;YAC5B,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;SAC1D;aAAM;YACH,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;SACrD;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,SAAS,2BAA2B,CAAC,IAAI,EAAE,GAAG;IAC1C,OAAO,GAAG,CAAC,mBAAmB,EAAE;WACzB,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AAC5C,CAAC;AAED;;;;;GAKG;AACH,SAAS,0BAA0B,CAAC,IAAI,EAAE,QAAQ;IAC9C,OAAO,KAAK,UAAU,eAAe,CAAC,MAAM;QACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI;YACA,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;SAC7B;QAAC,OAAO,GAAG,EAAE;YACV,mBAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;SACpG;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM;IACzC,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM;WACxB,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC;AAHD,4CAGC;AAED;;;;;GAKG;AACH,SAAS,2BAA2B,CAAC,IAAI,EAAE,QAAQ;IAC/C,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAI;IACxC,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACtB,MAAM,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,2CAA2C;QACxF,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,iCAAiC;QACxD,MAAM,CAAC,OAAO,GAAG;YACb,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAI;gBACb,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAI;gBAChB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACJ,CAAC;IACN,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACI,KAAK,UAAU,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY;IAC5D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,mBAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACjC,IAAI;YACA,MAAM,IAAI,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;SAC1B;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,gBAAgB,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE;gBACrE,mBAAG,CAAC,OAAO,CAAC,4BAA4B,QAAQ,+DAA+D;sBACzG,qHAAqH;sBACrH,uHAAuH,CAAC,CAAC;gBAC/H,gBAAgB,GAAG,KAAK,CAAC;aAC5B;YACD,mBAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;SACpF;KACJ;IACD,mBAAG,CAAC,KAAK,CAAC,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;AACvI,CAAC;AAtBD,sCAsBC;AAED,0BAA0B;AAC1B;;;;GAIG;AACH,SAAS,kCAAkC,CAAC,EAAE,EAAE,MAAM;IAClD,EAAE,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IAChC,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;IAClB,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACjB,MAAM,YAAY,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;IAChD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACnD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,KAAK,UAAU,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE;IACpF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC3B,IAAI,OAAO,CAAC;QACZ,IAAI,UAAU,CAAC;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtC,SAAS,aAAa,CAAC,MAAM;YACzB,IAAI,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;gBAAE,eAAe,EAAE,CAAC;QAC1D,CAAC;QAED,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,mBAAG,CAAC,KAAK,CAAC,oEAAoE,wBAAwB,MAAM;kBACtG,4GAA4G,CAAC,CAAC;YACpH,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAChD,IAAI,CAAC,cAAc,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;YACvD,OAAO,CAAC,cAAc,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC,wEAAwE;QACrG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAI,EAAE,QAAQ;IACrE,8BAA8B;IAC9B,0BAA0B;IAC1B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QAC1C,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;QACxC,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC,2CAA2C;QACxF,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1B,IAAI;YACA,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,mBAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;SACjF;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}

package/build/enqueue_links/enqueue_links.d.ts

@@ -1,117 +0,0 @@
-/**
- * The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)
- * either in a Puppeteer page, or in a Cheerio object (parsed HTML),
- * and enqueues the URLs in their `href` attributes to the provided {@link RequestQueue}.
- * If you're looking to find URLs in JavaScript heavy pages where links are not available
- * in `href` elements, but rather navigations are triggered in click handlers
- * see {@link puppeteer#enqueueLinksByClickingElements}.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await Apify.utils.enqueueLinks({
- *   page,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]',
- *       'https://www.example.com/purses/[.*]'
- *   ],
- * });
- * ```
- *
- * @param {object} options
- *   All `enqueueLinks()` parameters are passed
- *   via an options object with the following keys:
- * @param {PuppeteerPage|PlaywrightPage} [options.page]
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- *   Either `page` or `$` option must be provided.
- * @param {Number} [options.limit]
- *   Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.
- * @param {CheerioAPI} [options.$]
- *   [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.
- *   Either `page` or `$` option must be provided.
- * @param {RequestQueue} options.requestQueue
- *   A request queue to which the URLs will be enqueued.
- * @param {string} [options.selector='a']
- *   A CSS selector matching links to be enqueued.
- * @param {string} [options.baseUrl]
- *   A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
- *   since the relative URL resolution is done inside the browser automatically.
- * @param {Array<Object<string, *>>|Array<string>} [options.pseudoUrls]
- *   An array of {@link PseudoUrl}s matching the URLs to be enqueued,
- *   or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
- *
- *   The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.
- *   All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,
- *   which lets you specify special properties for the enqueued {@link Request} objects.
- *
- *   If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
- *   enqueues all links found on the page.
- * @param {RequestTransform} [options.transformRequestFunction]
- *   Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
- *   to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- *   when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- *   or to dynamically update or create `userData`.
- *
- *   For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
- *   when `uniqueKey` is computed.
- *
- *   **Example:**
- *   ```javascript
- *   {
- *       transformRequestFunction: (request) => {
- *           request.userData.foo = 'bar';
- *           request.keepUrlFragment = true;
- *           return request;
- *       }
- *   }
- *   ```
- * @return {Promise<Array<QueueOperationInfo>>}
- *   Promise that resolves to an array of {@link QueueOperationInfo} objects.
- * @memberOf utils
- * @name enqueueLinks
- * @function
- */
-export function enqueueLinks(options: {
-    page?: PuppeteerPage | PlaywrightPage | undefined;
-    limit?: number | undefined;
-    $?: CheerioAPI | undefined;
-    requestQueue: RequestQueue;
-    selector?: string | undefined;
-    baseUrl?: string | undefined;
-    pseudoUrls?: string[] | {
-        [x: string]: any;
-    }[] | undefined;
-    transformRequestFunction?: RequestTransform | undefined;
-}): Promise<Array<QueueOperationInfo>>;
-/**
- * Extracts URLs from a given Puppeteer Page.
- *
- * @param {PuppeteerPage|PlaywrightPage} page
- * @param {string} selector
- * @return {Promise<Array<string>>}
- * @ignore
- */
-export function extractUrlsFromPage(page: PuppeteerPage | PlaywrightPage, selector: string): Promise<Array<string>>;
-/**
- * Extracts URLs from a given Cheerio object.
- *
- * @param {CheerioAPI} $
- * @param {string} selector
- * @param {string} baseUrl
- * @return {string[]}
- * @ignore
- */
-export function extractUrlsFromCheerio($: CheerioAPI, selector: string, baseUrl: string): string[];
-// @ts-ignore optional peer dependency
-import { Page as PuppeteerPage } from "puppeteer";
-import { Page as PlaywrightPage } from "playwright-core";
-import { CheerioAPI } from "cheerio/lib/load";
-import { RequestQueue } from "../storages/request_queue";
-import { RequestTransform } from "./shared";
-import { QueueOperationInfo } from "../storages/request_queue";
-//# sourceMappingURL=enqueue_links.d.ts.map

package/build/enqueue_links/enqueue_links.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.js"],"names":[],"mappings":"AAkBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH;IAlDkD,IAAI;IAG1B,KAAK;IAED,CAAC;IAGA,YAAY,EAAlC,YAAY;IAEK,QAAQ;IAER,OAAO;IAGyB,UAAU;;;IAUhC,wBAAwB;IAmBlD,QAAQ,MAAM,kBAAkB,CAAC,CAAC,CAsD7C;AAED;;;;;;;GAOG;AACH,0CALW,aAAa,GAAC,cAAc,YAC5B,MAAM,GACL,QAAQ,MAAM,MAAM,CAAC,CAAC,CAMjC;AAED;;;;;;;;GAQG;AACH,0CANW,UAAU,YACV,MAAM,WACN,MAAM,GACL,MAAM,EAAE,CAmBnB"}