@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.5

package/index.d.ts

@@ -0,0 +1,12 @@
+export * from '@crawlee/browser';
+export * from './internals/puppeteer-crawler';
+export * from './internals/puppeteer-launcher';
+export * as puppeteerStealth from './internals/stealth';
+export type { StealthOptions } from './internals/stealth';
+export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception';
+export type { InterceptHandler } from './internals/utils/puppeteer_request_interception';
+export * as puppeteerUtils from './internals/utils/puppeteer_utils';
+export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils';
+export * as puppeteerClickElements from './internals/enqueue-links/click-elements';
+export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements';
+//# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gCAAgC,CAAC;AAE/C,OAAO,KAAK,gBAAgB,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,4BAA4B,MAAM,kDAAkD,CAAC;AACjG,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AAEzF,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAC;AACpE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,sBAAsB,MAAM,0CAA0C,CAAC;AACnF,YAAY,EAAE,qCAAqC,EAAE,MAAM,0CAA0C,CAAC"}

package/index.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.puppeteerClickElements = exports.puppeteerUtils = exports.puppeteerRequestInterception = exports.puppeteerStealth = void 0;
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("@crawlee/browser"), exports);
+tslib_1.__exportStar(require("./internals/puppeteer-crawler"), exports);
+tslib_1.__exportStar(require("./internals/puppeteer-launcher"), exports);
+exports.puppeteerStealth = tslib_1.__importStar(require("./internals/stealth"));
+exports.puppeteerRequestInterception = tslib_1.__importStar(require("./internals/utils/puppeteer_request_interception"));
+exports.puppeteerUtils = tslib_1.__importStar(require("./internals/utils/puppeteer_utils"));
+exports.puppeteerClickElements = tslib_1.__importStar(require("./internals/enqueue-links/click-elements"));
+//# sourceMappingURL=index.js.map

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,wEAA8C;AAC9C,yEAA+C;AAE/C,gFAAwD;AAGxD,yHAAiG;AAGjG,4FAAoE;AAWpE,2GAAmF"}

package/index.mjs

@@ -0,0 +1,76 @@
+import mod from "./index.js";
+
+export default mod;
+export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
+export const AutoscaledPool = mod.AutoscaledPool;
+export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
+export const BasicCrawler = mod.BasicCrawler;
+export const BrowserCrawler = mod.BrowserCrawler;
+export const BrowserLauncher = mod.BrowserLauncher;
+export const Configuration = mod.Configuration;
+export const CookieParseError = mod.CookieParseError;
+export const CrawlerExtension = mod.CrawlerExtension;
+export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
+export const Dataset = mod.Dataset;
+export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
+export const EnqueueStrategy = mod.EnqueueStrategy;
+export const EventManager = mod.EventManager;
+export const EventType = mod.EventType;
+export const KeyValueStore = mod.KeyValueStore;
+export const LocalEventManager = mod.LocalEventManager;
+export const Log = mod.Log;
+export const LogLevel = mod.LogLevel;
+export const Logger = mod.Logger;
+export const LoggerJson = mod.LoggerJson;
+export const LoggerText = mod.LoggerText;
+export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
+export const ProxyConfiguration = mod.ProxyConfiguration;
+export const PuppeteerCrawler = mod.PuppeteerCrawler;
+export const PuppeteerLauncher = mod.PuppeteerLauncher;
+export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
+export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
+export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
+export const Request = mod.Request;
+export const RequestList = mod.RequestList;
+export const RequestQueue = mod.RequestQueue;
+export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
+export const STATUS_CODES_BLOCKED = mod.STATUS_CODES_BLOCKED;
+export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
+export const Session = mod.Session;
+export const SessionPool = mod.SessionPool;
+export const Snapshotter = mod.Snapshotter;
+export const Statistics = mod.Statistics;
+export const StorageManager = mod.StorageManager;
+export const SystemStatus = mod.SystemStatus;
+export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
+export const checkAndSerialize = mod.checkAndSerialize;
+export const chunkBySize = mod.chunkBySize;
+export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
+export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
+export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
+export const createDeserialize = mod.createDeserialize;
+export const createRequestOptions = mod.createRequestOptions;
+export const createRequests = mod.createRequests;
+export const deserializeArray = mod.deserializeArray;
+export const diffCookies = mod.diffCookies;
+export const enqueueLinks = mod.enqueueLinks;
+export const getCookiesFromResponse = mod.getCookiesFromResponse;
+export const getRequestId = mod.getRequestId;
+export const handleRequestTimeout = mod.handleRequestTimeout;
+export const launchPuppeteer = mod.launchPuppeteer;
+export const log = mod.log;
+export const maybeStringify = mod.maybeStringify;
+export const mergeCookies = mod.mergeCookies;
+export const openDataset = mod.openDataset;
+export const puppeteerClickElements = mod.puppeteerClickElements;
+export const puppeteerRequestInterception = mod.puppeteerRequestInterception;
+export const puppeteerStealth = mod.puppeteerStealth;
+export const puppeteerUtils = mod.puppeteerUtils;
+export const pushData = mod.pushData;
+export const resolveBaseUrl = mod.resolveBaseUrl;
+export const serializeArray = mod.serializeArray;
+export const storage = mod.storage;
+export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
+export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
+export const validateGlobPattern = mod.validateGlobPattern;
+export const validators = mod.validators;

package/internals/enqueue-links/click-elements.d.ts

@@ -0,0 +1,188 @@
+import { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, RequestQueue, storage } from '@crawlee/browser';
+import { Dictionary } from '@crawlee/utils';
+import { ClickOptions, Page, Target } from 'puppeteer';
+export interface EnqueueLinksByClickingElementsOptions {
+    /**
+     * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+     */
+    page: Page;
+    /**
+     * A request queue to which the URLs will be enqueued.
+     */
+    requestQueue: RequestQueue;
+    /**
+     * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
+     * value. This is to prevent suboptimal use of this function by using it too broadly.
+     */
+    selector: string;
+    /**
+     * Click options for use in Puppeteer's click handler.
+     */
+    clickOptions?: ClickOptions;
+    /**
+     * An array of glob pattern strings or plain objects
+     * containing glob pattern strings matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * The matching is always case-insensitive.
+     * If you need case-sensitive matching, use `regexps` property directly.
+     *
+     * If `globs` is an empty array or `undefined`, then the function
+     * enqueues all the intercepted navigation requests produced by the page
+     * after clicking on elements matching the provided CSS selector.
+     */
+    globs?: GlobInput[];
+    /**
+     * An array of regular expressions or plain objects
+     * containing regular expressions matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `regexp` property, which holds the regular expression.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * If `regexps` is an empty array or `undefined`, then the function
+     * enqueues all the intercepted navigation requests produced by the page
+     * after clicking on elements matching the provided CSS selector.
+     */
+    regexps?: RegExpInput[];
+    /**
+     * *NOTE:* In future versions of SDK the options will be removed.
+     * Please use `globs` or `regexps` instead.
+     *
+     * An array of {@link PseudoUrl} strings or plain objects
+     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * With a pseudo-URL string, the matching is always case-insensitive.
+     * If you need case-sensitive matching, use `regexps` property directly.
+     *
+     * If `pseudoUrls` is an empty array or `undefined`, then the function
+     * enqueues all the intercepted navigation requests produced by the page
+     * after clicking on elements matching the provided CSS selector.
+     *
+     * @deprecated prefer using `globs` or `regexps` instead
+     */
+    pseudoUrls?: PseudoUrlInput[];
+    /**
+     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
+     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
+     * or to dynamically update or create `userData`.
+     *
+     * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
+     * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
+     * (POST requests).
+     *
+     * **Example:**
+     * ```javascript
+     * {
+     *     transformRequestFunction: (request) => {
+     *         request.userData.foo = 'bar';
+     *         request.useExtendedUniqueKey = true;
+     *         return request;
+     *     }
+     * }
+     * ```
+     */
+    transformRequestFunction?: RequestTransform;
+    /**
+     * Clicking in the page triggers various asynchronous operations that lead to new URLs being shown
+     * by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser.
+     * These events often happen only some time after the actual click. Requests typically take milliseconds
+     * while new tabs open in hundreds of milliseconds.
+     *
+     * To be able to capture all those events, the `enqueueLinksByClickingElements()` function repeatedly waits
+     * for the `waitForPageIdleSecs`. By repeatedly we mean that whenever a relevant event is triggered, the timer
+     * is restarted. As long as new events keep coming, the function will not return, unless
+     * the below `maxWaitForPageIdleSecs` timeout is reached.
+     *
+     * You may want to reduce this for example when you're sure that your clicks do not open new tabs,
+     * or increase when you're not getting all the expected URLs.
+     * @default 1
+     */
+    waitForPageIdleSecs?: number;
+    /**
+     * This is the maximum period for which the function will keep tracking events, even if more events keep coming.
+     * Its purpose is to prevent a deadlock in the page by periodic events, often unrelated to the clicking itself.
+     * See `waitForPageIdleSecs` above for an explanation.
+     * @default 5
+     */
+    maxWaitForPageIdleSecs?: number;
+}
+/**
+ * The function finds elements matching a specific CSS selector in a Puppeteer page,
+ * clicks all those elements using a mouse move and a left mouse button click and intercepts
+ * all the navigation requests that are subsequently produced by the page. The intercepted
+ * requests, including their methods, headers and payloads are then enqueued to a provided
+ * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available
+ * in `href` elements, but rather navigations are triggered in click handlers.
+ * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
+ *
+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
+ * and override settings of the enqueued {@link Request} objects.
+ *
+ * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,
+ * it is recommended to only use this function as the last operation in the page.
+ *
+ * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements
+ * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.
+ *
+ * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation
+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting
+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements
+ * that you assume or know will produce a navigation. You can certainly click everything by using
+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.
+ *
+ * **Example usage**
+ *
+ * ```javascript
+ * await Actor.utils.puppeteer.enqueueLinksByClickingElements({
+ *   page,
+ *   requestQueue,
+ *   selector: 'a.product-detail',
+ *   pseudoUrls: [
+ *       'https://www.example.com/handbags/[.*]'
+ *       'https://www.example.com/purses/[.*]'
+ *   ],
+ * });
+ * ```
+ *
+ * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
+ */
+export declare function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<storage.BatchAddRequestsResult>;
+interface WaitForPageIdleOptions {
+    page: Page;
+    waitForPageIdleMillis?: number;
+    maxWaitForPageIdleMillis?: number;
+}
+interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPageIdleOptions {
+    selector: string;
+    clickOptions?: ClickOptions;
+}
+/**
+ * Clicks all elements of given page matching given selector.
+ * Catches and intercepts all initiated navigation requests and opened pages.
+ * Returns a list of all target URLs.
+ * @ignore
+ */
+export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
+/**
+ * We're only interested in pages created by the page we're currently clicking in.
+ * There will generally be a lot of other targets being created in the browser.
+ */
+export declare function isTargetRelevant(page: Page, target: Target): boolean;
+/**
+ * Click all elements matching the given selector. To be able to do this using
+ * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
+ * so we first move them to the top of the page's stacking context and then click.
+ * We do all in series to prevent elements from hiding one another. Therefore,
+ * for large element sets, this will take considerable amount of time.
+ * @ignore
+ */
+export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+export {};
+//# sourceMappingURL=click-elements.d.ts.map

package/internals/enqueue-links/click-elements.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEhB,YAAY,EACZ,OAAO,EACV,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,OAAO,EAGH,YAAY,EAGZ,IAAI,EAEJ,MAAM,EACT,MAAM,WAAW,CAAC;AAOnB,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAAC,OAAO,EAAE,qCAAqC,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAmE5I;AAED,UAAU,sBAAsB;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,UAAU,kDAAmD,SAAQ,sBAAsB;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAsB,2CAA2C,CAAC,OAAO,EAAE,kDAAkD,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAiCpJ;AAoDD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAGpE;AAiDD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAuB5G"}

package/internals/enqueue-links/click-elements.js

@@ -0,0 +1,336 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clickElements = exports.isTargetRelevant = exports.clickElementsAndInterceptNavigationRequests = exports.enqueueLinksByClickingElements = void 0;
+const tslib_1 = require("tslib");
+const browser_1 = require("@crawlee/browser");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const url_1 = require("url");
+const puppeteer_request_interception_1 = require("../utils/puppeteer_request_interception");
+const STARTING_Z_INDEX = 2147400000;
+const log = log_1.default.child({ prefix: 'Puppeteer Click Elements' });
+/**
+ * The function finds elements matching a specific CSS selector in a Puppeteer page,
+ * clicks all those elements using a mouse move and a left mouse button click and intercepts
+ * all the navigation requests that are subsequently produced by the page. The intercepted
+ * requests, including their methods, headers and payloads are then enqueued to a provided
+ * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available
+ * in `href` elements, but rather navigations are triggered in click handlers.
+ * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
+ *
+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
+ * and override settings of the enqueued {@link Request} objects.
+ *
+ * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,
+ * it is recommended to only use this function as the last operation in the page.
+ *
+ * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements
+ * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.
+ *
+ * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation
+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting
+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements
+ * that you assume or know will produce a navigation. You can certainly click everything by using
+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.
+ *
+ * **Example usage**
+ *
+ * ```javascript
+ * await Actor.utils.puppeteer.enqueueLinksByClickingElements({
+ *   page,
+ *   requestQueue,
+ *   selector: 'a.product-detail',
+ *   pseudoUrls: [
+ *       'https://www.example.com/handbags/[.*]'
+ *       'https://www.example.com/purses/[.*]'
+ *   ],
+ * });
+ * ```
+ *
+ * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
+ */
+async function enqueueLinksByClickingElements(options) {
+    (0, ow_1.default)(options, ow_1.default.object.exactShape({
+        page: ow_1.default.object.hasKeys('goto', 'evaluate'),
+        requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
+        selector: ow_1.default.string,
+        clickOptions: ow_1.default.optional.object.hasKeys('clickCount', 'delay'),
+        pseudoUrls: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('purl'))),
+        globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
+        regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
+        transformRequestFunction: ow_1.default.optional.function,
+        waitForPageIdleSecs: ow_1.default.optional.number,
+        maxWaitForPageIdleSecs: ow_1.default.optional.number,
+    }));
+    const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, } = options;
+    const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
+    const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const urlPatternObjects = [];
+    if (pseudoUrls?.length) {
+        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        urlPatternObjects.push(...(0, browser_1.constructRegExpObjectsFromPseudoUrls)(pseudoUrls));
+    }
+    if (globs?.length) {
+        urlPatternObjects.push(...(0, browser_1.constructGlobObjectsFromGlobs)(globs));
+    }
+    if (regexps?.length) {
+        urlPatternObjects.push(...(0, browser_1.constructRegExpObjectsFromRegExps)(regexps));
+    }
+    const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
+        page,
+        selector,
+        waitForPageIdleMillis,
+        maxWaitForPageIdleMillis,
+        clickOptions,
+    });
+    let requestOptions = (0, browser_1.createRequestOptions)(interceptedRequests);
+    if (transformRequestFunction) {
+        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
+    }
+    const requests = (0, browser_1.createRequests)(requestOptions, urlPatternObjects);
+    return requestQueue.addRequests(requests);
+}
+exports.enqueueLinksByClickingElements = enqueueLinksByClickingElements;
+/**
+ * Clicks all elements of given page matching given selector.
+ * Catches and intercepts all initiated navigation requests and opened pages.
+ * Returns a list of all target URLs.
+ * @ignore
+ */
+async function clickElementsAndInterceptNavigationRequests(options) {
+    const { page, selector, waitForPageIdleMillis, maxWaitForPageIdleMillis, clickOptions, } = options;
+    const uniqueRequests = new Set();
+    const browser = page.browser();
+    const onInterceptedRequest = createInterceptRequestHandler(page, uniqueRequests);
+    const onTargetCreated = createTargetCreatedHandler(page, uniqueRequests);
+    const onFrameNavigated = createFrameNavigatedHandler(page, uniqueRequests);
+    await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, onInterceptedRequest);
+    browser.on("targetcreated" /* TargetCreated */, onTargetCreated);
+    page.on("framenavigated" /* FrameNavigated */, onFrameNavigated);
+    await preventHistoryNavigation(page);
+    await clickElements(page, selector, clickOptions);
+    await waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis });
+    await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
+    browser.off("targetcreated" /* TargetCreated */, onTargetCreated);
+    page.off("framenavigated" /* FrameNavigated */, onFrameNavigated);
+    await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, onInterceptedRequest);
+    const serializedRequests = Array.from(uniqueRequests);
+    return serializedRequests.map((r) => JSON.parse(r));
+}
+exports.clickElementsAndInterceptNavigationRequests = clickElementsAndInterceptNavigationRequests;
+/**
+ * @ignore
+ */
+function createInterceptRequestHandler(page, requests) {
+    return async function onInterceptedRequest(req) {
+        if (!isTopFrameNavigationRequest(page, req))
+            return req.continue();
+        const url = req.url();
+        requests.add(JSON.stringify({
+            url,
+            headers: req.headers(),
+            method: req.method(),
+            payload: req.postData(),
+        }));
+        if (req.redirectChain().length) {
+            await req.respond({ body: '' }); // Prevents 301/302 redirect
+        }
+        else {
+            await req.abort('aborted'); // Prevents navigation by js
+        }
+    };
+}
+/**
+ * @ignore
+ */
+function isTopFrameNavigationRequest(page, req) {
+    return req.isNavigationRequest()
+        && req.frame() === page.mainFrame();
+}
+/**
+ * @ignore
+ */
+function createTargetCreatedHandler(page, requests) {
+    return async function onTargetCreated(target) {
+        if (!isTargetRelevant(page, target))
+            return;
+        const url = target.url();
+        requests.add(JSON.stringify({ url }));
+        // We want to close the page but don't care about
+        // possible errors like target closed.
+        try {
+            const createdPage = await target.page();
+            await createdPage.close();
+        }
+        catch (err) {
+            log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
+        }
+    };
+}
+/**
+ * We're only interested in pages created by the page we're currently clicking in.
+ * There will generally be a lot of other targets being created in the browser.
+ */
+function isTargetRelevant(page, target) {
+    return target.type() === 'page'
+        && page.target() === target.opener();
+}
+exports.isTargetRelevant = isTargetRelevant;
+/**
+ * @ignore
+ */
+function createFrameNavigatedHandler(page, requests) {
+    return function onFrameNavigated(frame) {
+        if (frame !== page.mainFrame())
+            return;
+        const url = frame.url();
+        requests.add(JSON.stringify({ url }));
+    };
+}
+/**
+ * @ignore
+ */
+async function preventHistoryNavigation(page) {
+    /* istanbul ignore next */
+    return page.evaluate(() => {
+        window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangle
+        delete window.history; // Simple override does not work.
+        window.history = {
+            stateHistory: [],
+            length: 0,
+            state: {},
+            go() { },
+            back() { },
+            forward() { },
+            pushState(...args) {
+                this.stateHistory.push(args);
+            },
+            replaceState(...args) {
+                this.stateHistory.push(args);
+            },
+        };
+    });
+}
+/**
+ * Click all elements matching the given selector. To be able to do this using
+ * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
+ * so we first move them to the top of the page's stacking context and then click.
+ * We do all in series to prevent elements from hiding one another. Therefore,
+ * for large element sets, this will take considerable amount of time.
+ * @ignore
+ */
+async function clickElements(page, selector, clickOptions) {
+    const elementHandles = await page.$$(selector);
+    log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
+    let clickedElementsCount = 0;
+    let zIndex = STARTING_Z_INDEX;
+    let shouldLogWarning = true;
+    for (const handle of elementHandles) {
+        try {
+            await page.evaluate(updateElementCssToEnableMouseClick, handle, zIndex++);
+            await handle.click(clickOptions);
+            clickedElementsCount++;
+        }
+        catch (err) {
+            const e = err;
+            if (shouldLogWarning && e.stack.includes('is detached from document')) {
+                log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. `
+                    + 'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. '
+                    + 'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
+                shouldLogWarning = false;
+            }
+            log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
+        }
+    }
+    log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
+}
+exports.clickElements = clickElements;
+/* istanbul ignore next */
+/**
+ * This is an in browser function!
+ */
+function updateElementCssToEnableMouseClick(el, zIndex) {
+    el.style.visibility = 'visible';
+    el.style.display = 'block';
+    el.style.position = 'fixed';
+    el.style.zIndex = String(zIndex);
+    el.style.left = '0';
+    el.style.top = '0';
+    const boundingRect = el.getBoundingClientRect();
+    if (!boundingRect.height)
+        el.style.height = '10px';
+    if (!boundingRect.width)
+        el.style.width = '10px';
+}
+/**
+ * This function tracks whether any requests, frame navigations or targets were emitted
+ * in the past idleIntervalMillis and whenever the interval registers no activity,
+ * the function returns.
+ *
+ * It will also return when a final timeout, represented by the timeoutMillis parameter
+ * is reached, to prevent blocking on pages with constant network activity.
+ *
+ * We need this to make sure we don't finish too soon when intercepting requests triggered
+ * by clicking in the page. They often get registered by the Node.js process only some
+ * milliseconds after clicking and we would lose those requests. This is especially prevalent
+ * when there's only a single element to click.
+ * @ignore
+ */
+async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis }) {
+    return new Promise((resolve) => {
+        let timeout;
+        let maxTimeout;
+        const context = page.browserContext();
+        function newTabTracker(target) {
+            if (isTargetRelevant(page, target))
+                activityHandler();
+        }
+        function activityHandler() {
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
+                clearTimeout(maxTimeout);
+                finish();
+            }, waitForPageIdleMillis);
+        }
+        function maxTimeoutHandler() {
+            log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. `
+                + 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
+            finish();
+        }
+        function finish() {
+            page.off("request" /* Request */, activityHandler);
+            page.off("framenavigated" /* FrameNavigated */, activityHandler);
+            context.off("targetcreated" /* TargetCreated */, newTabTracker);
+            resolve();
+        }
+        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        activityHandler(); // We call this once manually in case there would be no requests at all.
+        page.on("request" /* Request */, activityHandler);
+        page.on("framenavigated" /* FrameNavigated */, activityHandler);
+        context.on("targetcreated" /* TargetCreated */, newTabTracker);
+    });
+}
+/**
+ * @ignore
+ */
+async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
+    /* eslint-disable no-shadow */
+    /* istanbul ignore next */
+    const state = await page.evaluate(() => {
+        const { stateHistory } = window.history;
+        window.history = window.__originalHistory__; // eslint-disable-line no-underscore-dangle
+        return stateHistory;
+    });
+    state.forEach((args) => {
+        try {
+            const stateUrl = args[args.length - 1];
+            const url = new url_1.URL(stateUrl, page.url()).href;
+            requests.add(JSON.stringify({ url }));
+        }
+        catch (err) {
+            log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
+        }
+    });
+}
+//# sourceMappingURL=click-elements.js.map

package/internals/enqueue-links/click-elements.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":";;;;AAAA,8CAa0B;AAC1B,6DAA8B;AAE9B,oDAAoB;AAWpB,6BAA0B;AAC1B,4FAAoH;AAEpH,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AA2H/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACI,KAAK,UAAU,8BAA8B,CAAC,OAA8C;IAC/F,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,QAAQ,EAAE,YAAE,CAAC,MAAM;QACnB,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7C,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,GAC7B,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,8CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,uCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,2CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,IAAA,8BAAoB,EAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpF;IACD,MAAM,QAAQ,GAAG,IAAA,wBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACnE,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAnED,wEAmEC;AAaD;;;;;GAKG;AACI,KAAK,UAAU,2CAA2C,CAAC,OAA2D;IACzH,MAAM,EACF,IAAI,EACJ,QAAQ,EACR,qBAAqB,EACrB,wBAAwB,EACxB,YAAY,GACf,GAAG,OAAO,CAAC;IAEZ,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,eAAe,GAAG,0BAA0B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,sCAAqC,eAAe,CAAC,CAAC;IAChE,IAAI,CAAC,EAAE,wCAAmC,gBAAgB,CAAC,CAAC;IAE5D,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,sCAAqC,eAAe,CAAC,CAAC;IACjE,IAAI,CAAC,GAAG,wCAAmC,gBAAgB,CAAC,CAAC;IAC7D,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAEhE,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAjCD,kGAiCC;AAED;;GAEG;AACH,SAAS,6BAA6B,CAAC,IAAU,EAAE,QAAqB;IACpE,OAAO,KAAK,UAAU,oBAAoB,CAAC,GAAG;QAC1C,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACxB,GAAG;YACH,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;YACpB,OAAO,EAAE,GAAG,CAAC,QAAQ,EAAE;SAC1B,CAAC,CAAC,CAAC;QAEJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE;YAC5B,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;SAChE;aAAM;YACH,MAAM,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;SAC3D;IACL,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,GAAqB;IAClE,OAAO,GAAG,CAAC,mBAAmB,EAAE;WACzB,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,IAAU,EAAE,QAAqB;IACjE,OAAO,KAAK,UAAU,eAAe,CAAC,MAAM;QACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;QACzB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI;YACA,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,WAAY,CAAC,KAAK,EAAE,CAAC;SAC9B;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC/G;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,IAAU,EAAE,MAAc;IACvD,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM;WACxB,IAAI,CAAC,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC;AAHD,4CAGC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,QAAqB;IAClE,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAaD;;GAEG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAU;IAC9C,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrB,MAAgC,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,2CAA2C;QACnH,OAAQ,MAAgC,CAAC,OAAO,CAAC,CAAC,iCAAiC;QAClF,MAAgC,CAAC,OAAO,GAAG;YACxC,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAe;gBACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAe;gBAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACW,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,YAA2B;IACzF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACjC,IAAI;YACA,MAAM,IAAI,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;SAC1B;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,gBAAgB,IAAI,CAAC,CAAC,KAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE;gBACpE,GAAG,CAAC,OAAO,CAAC,4BAA4B,QAAQ,+DAA+D;sBACzG,qHAAqH;sBACrH,uHAAuH,CAAC,CAAC;gBAC/H,gBAAgB,GAAG,KAAK,CAAC;aAC5B;YACD,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SAClF;KACJ;IACD,GAAG,CAAC,KAAK,CAAC,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;AACvI,CAAC;AAvBD,sCAuBC;AAED,0BAA0B;AAC1B;;GAEG;AACH,SAAS,kCAAkC,CAAC,EAAe,EAAE,MAAc;IACvE,EAAE,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IAChC,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjC,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACpB,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,MAAM,YAAY,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;IAChD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACnD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAA0B;IAC5G,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,IAAI,OAAuB,CAAC;QAC5B,IAAI,UAA0B,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtC,SAAS,aAAa,CAAC,MAAc;YACjC,IAAI,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;gBAAE,eAAe,EAAE,CAAC;QAC1D,CAAC;QAED,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,GAAG,CAAC,KAAK,CAAC,oEAAoE,wBAAwB,MAAM;kBACtG,4GAA4G,CAAC,CAAC;YACpH,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,GAAG,0BAA4B,eAAe,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,wCAAmC,eAAe,CAAC,CAAC;YAC5D,OAAO,CAAC,GAAG,sCAA4C,aAAa,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,eAAe,EAAE,CAAC,CAAC,wEAAwE;QAC3F,IAAI,CAAC,EAAE,0BAA4B,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE,wCAAmC,eAAe,CAAC,CAAC;QAC3D,OAAO,CAAC,EAAE,sCAA4C,aAAa,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAU,EAAE,QAAqB;IACxF,8BAA8B;IAC9B,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAiC,CAAC;QACjE,MAAgC,CAAC,OAAO,GAAI,MAAgC,CAAC,mBAAmB,CAAC,CAAC,2CAA2C;QAC9I,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI;YACA,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;SAC5F;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}