@crawlee/jsdom 4.0.0-beta.12 → 4.0.0-beta.121

package/internals/jsdom-crawler.js

@@ -1,6 +1,5 @@
-import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
+import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
 import { sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
 import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
 import ow from 'ow';
 import { addTimeoutToPromise } from '@apify/timeout';
@@ -26,38 +25,40 @@ import { addTimeoutToPromise } from '@apify/timeout';
  * and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
  * using the `window` object.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link JSDOMCrawlerOptions.requestList}
- * or {@link JSDOMCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link JSDOMCrawlerOptions.requestList} and {@link JSDOMCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, `JSDOMCrawler` only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
  * For more details, see {@link JSDOMCrawlerOptions.requestHandler}.
  *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -88,25 +89,29 @@ export class JSDOMCrawler extends HttpCrawler {
         runScripts: ow.optional.boolean,
         hideInternalConsole: ow.optional.boolean,
     };
-    runScripts;
-    hideInternalConsole;
-    virtualConsole = null;
-    constructor(options = {}, config) {
-        const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
+    #runScripts;
+    #hideInternalConsole;
+    #virtualConsole = null;
+    constructor(options = {}) {
+        const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
         super({
             ...httpOptions,
-            contextPipelineBuilder: () => this.buildContextPipeline()
-                .compose({
-                action: async (context) => await this.parseContent(context),
-                cleanup: async (context) => {
-                    this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
-                    context.window?.close();
-                },
-            })
-                .compose({ action: async (context) => await this.addHelpers(context) }),
-        }, config);
-        this.runScripts = runScripts;
-        this.hideInternalConsole = hideInternalConsole;
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
+        this.#runScripts = runScripts;
+        this.#hideInternalConsole = hideInternalConsole;
+    }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose({
+            action: async (context) => await this.parseContent(context),
+            cleanup: async (context) => {
+                this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
+                context.window?.close();
+            },
+        })
+            .compose({ action: async (context) => await this.addHelpers(context) });
     }
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
@@ -123,86 +128,108 @@ export class JSDOMCrawler extends HttpCrawler {
      * ```
      */
     getVirtualConsole() {
-        if (this.virtualConsole) {
-            return this.virtualConsole;
+        if (this.#virtualConsole) {
+            return this.#virtualConsole;
         }
-        this.virtualConsole = new VirtualConsole();
-        if (!this.hideInternalConsole) {
-            this.virtualConsole.sendTo(console, { omitJSDOMErrors: true });
+        this.#virtualConsole = new VirtualConsole();
+        if (!this.#hideInternalConsole) {
+            this.#virtualConsole.sendTo(console, { omitJSDOMErrors: true });
         }
-        this.virtualConsole.on('jsdomError', this.jsdomErrorHandler);
-        return this.virtualConsole;
+        this.#virtualConsole.on('jsdomError', this.jsdomErrorHandler);
+        return this.#virtualConsole;
     }
-    jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', error);
+    jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
     async parseContent(crawlingContext) {
-        const isXml = crawlingContext.contentType.type.includes('xml');
-        // TODO handle non-string
-        const { window } = new JSDOM(crawlingContext.body.toString(), {
-            url: crawlingContext.response.url,
-            contentType: isXml ? 'text/xml' : 'text/html',
-            runScripts: this.runScripts ? 'dangerously' : undefined,
-            resources,
-            virtualConsole: this.getVirtualConsole(),
-            pretendToBeVisual: true,
-        });
-        // add some stubs in place of missing API so processing won't fail
-        Object.defineProperty(window, 'matchMedia', {
-            writable: true,
-            value: (query) => ({
-                matches: false,
-                media: query,
-                onchange: null,
-                addListener: () => { },
-                removeListener: () => { },
-                addEventListener: () => { },
-                removeEventListener: () => { },
-                dispatchEvent: () => { },
-            }),
-        });
-        window.document.createRange = () => {
-            const range = new window.Range();
-            range.getBoundingClientRect = () => ({});
-            range.getClientRects = () => ({ item: () => null, length: 0 });
-            return range;
-        };
-        if (this.runScripts) {
-            try {
-                await addTimeoutToPromise(async () => {
-                    return new Promise((resolve) => {
-                        window.addEventListener('load', () => {
-                            resolve();
-                        }, false);
-                    }).catch();
-                }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
+        try {
+            const isXml = crawlingContext.contentType.type.includes('xml');
+            // TODO handle non-string
+            const { window } = new JSDOM(crawlingContext.body.toString(), {
+                url: crawlingContext.response.url,
+                contentType: isXml ? 'text/xml' : 'text/html',
+                runScripts: this.#runScripts ? 'dangerously' : undefined,
+                resources,
+                virtualConsole: this.getVirtualConsole(),
+                pretendToBeVisual: true,
+            });
+            // add some stubs in place of missing API so processing won't fail
+            Object.defineProperty(window, 'matchMedia', {
+                writable: true,
+                value: (query) => ({
+                    matches: false,
+                    media: query,
+                    onchange: null,
+                    addListener: () => { },
+                    removeListener: () => { },
+                    addEventListener: () => { },
+                    removeEventListener: () => { },
+                    dispatchEvent: () => { },
+                }),
+            });
+            window.document.createRange = () => {
+                const range = new window.Range();
+                range.getBoundingClientRect = () => ({});
+                range.getClientRects = () => ({ item: () => null, length: 0 });
+                return range;
+            };
+            if (this.#runScripts) {
+                try {
+                    await addTimeoutToPromise(async () => {
+                        return new Promise((resolve) => {
+                            window.addEventListener('load', () => {
+                                resolve();
+                            }, false);
+                        }).catch();
+                    }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
+                }
+                catch (e) {
+                    this.log.debug(e.message);
+                }
             }
-            catch (e) {
-                this.log.debug(e.message);
+            return {
+                window,
+                get body() {
+                    return window.document.documentElement.outerHTML;
+                },
+                get document() {
+                    return window.document;
+                },
+            };
+        }
+        catch (err) {
+            if (err instanceof NavigationSkippedError) {
+                return {
+                    get window() {
+                        throw new NavigationSkippedError('The `window` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                    get body() {
+                        throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                    get document() {
+                        throw new NavigationSkippedError('The `document` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                };
             }
+            throw err;
         }
-        return {
-            window,
-            get body() {
-                return window.document.documentElement.outerHTML;
-            },
-            get document() {
-                return window.document;
-            },
-        };
     }
     async addHelpers(crawlingContext) {
         return {
             enqueueLinks: async (enqueueOptions) => {
-                return domCrawlerEnqueueLinks({
-                    options: enqueueOptions,
+                return (await domCrawlerEnqueueLinks({
+                    options: {
+                        ...enqueueOptions,
+                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
+                    },
                     window: crawlingContext.window,
-                    requestQueue: await this.getRequestQueue(),
+                    requestManager: await this.getRequestManager(),
                     robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.onSkippedRequest,
+                    onSkippedRequest: this.handleSkippedRequest,
                     originalRequestUrl: crawlingContext.request.url,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
-                });
+                })); // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
             },
             async waitForSelector(selector, timeoutMs = 5_000) {
+                const cheerio = await import('cheerio');
                 const $ = cheerio.load(crawlingContext.body);
                 if ($(selector).get().length === 0) {
                     if (timeoutMs) {
@@ -214,6 +241,7 @@ export class JSDOMCrawler extends HttpCrawler {
                 }
             },
             async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                const cheerio = await import('cheerio');
                 const $ = cheerio.load(crawlingContext.body);
                 if (selector && $(selector).get().length === 0) {
                     throw new Error(`Selector '${selector}' not found.`);
@@ -224,24 +252,36 @@ export class JSDOMCrawler extends HttpCrawler {
     }
 }
 /** @internal */
-export async function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
+function containsEnqueueLinks(options) {
+    return !!options.enqueueLinks;
+}
+/** @internal */
+export async function domCrawlerEnqueueLinks(options) {
+    const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
     if (!window) {
         throw new Error('Cannot enqueue links because the JSDOM is not available.');
     }
     const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: options?.strategy,
+        enqueueStrategy: enqueueLinksOptions?.strategy,
         finalRequestUrl,
         originalRequestUrl,
-        userProvidedBaseUrl: options?.baseUrl,
+        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
     });
-    const urls = extractUrlsFromWindow(window, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    if (containsEnqueueLinks(options)) {
+        return options.enqueueLinks({
+            urls,
+            baseUrl,
+            ...enqueueLinksOptions,
+        });
+    }
     return enqueueLinks({
-        requestQueue,
-        robotsTxtFile,
-        onSkippedRequest,
+        requestManager: options.requestManager,
+        robotsTxtFile: options.robotsTxtFile,
+        onSkippedRequest: options.onSkippedRequest,
         urls,
         baseUrl,
-        ...options,
+        ...enqueueLinksOptions,
     });
 }
 /**
@@ -260,31 +300,6 @@ function extractUrlsFromWindow(window, selector, baseUrl) {
     })
         .filter((href) => href !== undefined && href !== '');
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
- * Defaults to the {@link JSDOMCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<JSDOMCrawlingContext>()`.
- *
- * ```ts
- * import { JSDOMCrawler, createJSDOMRouter } from 'crawlee';
- *
- * const router = createJSDOMRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new JSDOMCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createJSDOMRouter(routes) {
-    return Router.create(routes);
+export function createJSDOMRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=jsdom-crawler.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/jsdom",
-    "version": "4.0.0-beta.12",
+    "version": "4.0.0-beta.121",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -47,11 +47,11 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.2",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/http": "4.0.0-beta.12",
-        "@crawlee/types": "4.0.0-beta.12",
-        "@crawlee/utils": "4.0.0-beta.12",
+        "@apify/timeout": "^0.4.4",
+        "@apify/utilities": "^2.7.10",
+        "@crawlee/http": "4.0.0-beta.121",
+        "@crawlee/types": "4.0.0-beta.121",
+        "@crawlee/utils": "4.0.0-beta.121",
         "@types/jsdom": "^21.1.7",
         "cheerio": "^1.0.0",
         "jsdom": "^26.1.0",
@@ -65,5 +65,5 @@
             }
         }
     },
-    "gitHead": "06431733e5cfa6cc3a4c105427592f87424037ca"
+    "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
 }

package/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,8BAA8B,CAAC"}

package/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,8BAA8B,CAAC"}

package/internals/jsdom-crawler.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"jsdom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EACb,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,sBAAsB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,WAAW,EAId,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAS,MAAM,gBAAgB,CAAC;AAE7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAyB,cAAc,EAAE,MAAM,OAAO,CAAC;AAK9D,MAAM,MAAM,iBAAiB,CACzB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,WAAW,mBAAmB,CAChC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,EACnE,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,eAAe,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,MAAM,SAAS,CACjB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,oBAAoB,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACrD,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IAEnB,IAAI,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAkF7D,qBAAa,YAAY,CAAC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,CAAE,SAAQ,WAAW,CAC9G,oBAAoB,EACpB,eAAe,CAClB;IACG,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAQ;gBAE3C,OAAO,GAAE,mBAAmB,CAAC,eAAe,CAAM,EAAE,MAAM,CAAC,EAAE,aAAa;IAwBtF;;;;;;;;;;;;;OAaG;IACH,iBAAiB;IAgBjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAuE;YAE3F,YAAY;YAmEZ,UAAU;CAqC3B;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,sBAAsB,CAAC,EACzC,OAAO,EACP,MAAM,EACN,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DA0B7B;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAC7B,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}

package/internals/jsdom-crawler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"jsdom-crawler.js","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAaA,OAAO,EACH,YAAY,EACZ,WAAW,EACX,sCAAsC,EACtC,MAAM,EACN,cAAc,GACjB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAwC,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC9D,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAuErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;AACH,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC;IACjC,6EAA6E;IAC7E,kDAAkD;IAClD,SAAS,EACL,uHAAuH;CAC9H,CAAC,CAAC;AAEH,MAAM,OAAO,YAAkF,SAAQ,WAGtG;IACa,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,WAAW,CAAC,YAAY;QAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC/B,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KAC3C,CAAC;IAEQ,UAAU,CAAU;IACpB,mBAAmB,CAAU;IAC7B,cAAc,GAA0B,IAAI,CAAC;IAEvD,YAAY,UAAgD,EAAE,EAAE,MAAsB;QAClF,MAAM,EAAE,UAAU,GAAG,KAAK,EAAE,mBAAmB,GAAG,KAAK,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAEpF,KAAK,CACD;YACI,GAAG,WAAW;YACd,sBAAsB,EAAE,GAAG,EAAE,CACzB,IAAI,CAAC,oBAAoB,EAAE;iBACtB,OAAO,CAAC;gBACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;oBACnE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;gBAC5B,CAAC;aACJ,CAAC;iBACD,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;SAClF,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,iBAAiB;QACb,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,cAAc,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE7D,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAEgB,iBAAiB,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;IAEjG,KAAK,CAAC,YAAY,CAAC,eAA4C;QACnE,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAE/D,yBAAyB;QACzB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAC1D,GAAG,EAAE,eAAe,CAAC,QAAQ,CAAC,GAAG;YACjC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;YAC7C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YACvD,SAAS;YACT,cAAc,EAAE,IAAI,CAAC,iBAAiB,EAAE;YACxC,iBAAiB,EAAE,IAAI;SAC1B,CAAC,CAAC;QAEH,kEAAkE;QAClE,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE;YACxC,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,CAAC,KAAc,EAAO,EAAE,CAAC,CAAC;gBAC7B,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC;gBACrB,cAAc,EAAE,GAAG,EAAE,GAAE,CAAC;gBACxB,gBAAgB,EAAE,GAAG,EAAE,GAAE,CAAC;gBAC1B,mBAAmB,EAAE,GAAG,EAAE,GAAE,CAAC;gBAC7B,aAAa,EAAE,GAAG,EAAE,GAAE,CAAC;aAC1B,CAAC;SACL,CAAC,CAAC;QACH,MAAM,CAAC,QAAQ,CAAC,WAAW,GAAG,GAAG,EAAE;YAC/B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjC,KAAK,CAAC,qBAAqB,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,CAAQ,CAAC;YAChD,KAAK,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,IAAW,EAAE,MAAM,EAAE,CAAC,EAAE,CAAQ,CAAC;YAC7E,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC;gBACD,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE;oBACP,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBACjC,MAAM,CAAC,gBAAgB,CACnB,MAAM,EACN,GAAG,EAAE;4BACD,OAAO,EAAE,CAAC;wBACd,CAAC,EACD,KAAK,CACR,CAAC;oBACN,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBACf,CAAC,EACD,MAAM,EACN,+CAA+C,CAClD,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QAED,OAAO;YACH,MAAM;YACN,IAAI,IAAI;gBACJ,OAAO,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;YACrD,CAAC;YACD,IAAI,QAAQ;gBACR,OAAO,MAAM,CAAC,QAAQ,CAAC;YAC3B,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,eAAkF;QACvG,OAAO;YACH,YAAY,EAAE,KAAK,EAAE,cAAoC,EAAE,EAAE;gBACzD,OAAO,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,cAAc;oBACvB,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;oBACvC,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;YACD,KAAK,CAAC,eAAe,CAAC,QAAgB,EAAE,SAAS,GAAG,KAAK;gBACrD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7C,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,IAAI,SAAS,EAAE,CAAC;wBACZ,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;wBAChB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;wBAClE,OAAO;oBACX,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC;YACD,KAAK,CAAC,gBAAgB,CAAC,QAAiB,EAAE,UAAU,GAAG,KAAK;gBACxD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7C,IAAI,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC;SACJ,CAAC;IACN,CAAC;;AAaL,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EACzC,OAAO,EACP,MAAM,EACN,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GACW;IAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,qBAAqB,CAC9B,MAAM,EACN,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,YAAY,CAAC;QAChB,YAAY;QACZ,aAAa;QACb,gBAAgB;QAChB,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,MAAiB,EAAE,QAAgB,EAAE,OAAe;IAC/E,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;SACnD,GAAG,CAAC,CAAC,IAAwB,EAAE,EAAE;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAa,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,iBAAiB,CAG/B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}