@crawlee/jsdom 4.0.0-beta.168 → 4.0.0-beta.169

package/index.d.ts

@@ -1,2 +1,3 @@
 export * from '@crawlee/http';
 export * from './internals/jsdom-crawler.js';
+export type { JSDOMParseResult } from './internals/jsdom-parser.js';

package/internals/jsdom-crawler.d.ts

@@ -1,10 +1,9 @@
-import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
-import { HttpCrawler } from '@crawlee/http';
+import type { CrawlingContext, DOMCrawlingContext, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
+import { DOMCrawler } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
-import type { DOMWindow } from 'jsdom';
 import { VirtualConsole } from 'jsdom';
 import { z } from 'zod';
+import type { JSDOMParseResult } from './jsdom-parser.js';
 export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
@@ -23,49 +22,86 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat
 export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
 export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
-    window: DOMWindow;
-    document: Document;
-    body: string;
-    /**
-     * Wait for an element matching the selector to appear.
-     * Timeout defaults to 5s.
-     *
-     * **Example usage:**
-     * ```ts
-     * async requestHandler({ waitForSelector, parseWithCheerio }) {
-     *     await waitForSelector('article h1');
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-    /**
-     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
-     * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
-     *
-     * **Example usage:**
-     * ```javascript
-     * async requestHandler({ parseWithCheerio }) {
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
-    /**
-     * Extracts URLs from the parsed DOM, without adding them to the request queue.
-     */
-    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
-    /**
-     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
-     */
-    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+JSONData extends Dictionary = any> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {
 }
 export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
-export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+/**
+ * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
+ * [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `JSDOMCrawler` uses raw HTTP requests to download web pages,
+ * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
+ * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
+ * because it loads the pages using full-featured headless Chrome browser.
+ *
+ * Alternatively, you can use {@link JSDOMCrawlerOptions.runScripts} to run website scripts in Node.
+ * JSDOM does not implement all the standards, so websites can break.
+ *
+ * **Limitation**:
+ * This crawler does not support proxies and cookies yet (each open starts with empty cookie store), and the user agent is always set to `Chrome`.
+ *
+ * `JSDOMCrawler` downloads each URL using a plain HTTP request,
+ * parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom)
+ * and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
+ * using the `window` object.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
+ *
+ * > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
+ *
+ * ```
+ * preNavigationHooks: [
+ *     (crawlingContext) => {
+ *         // ...
+ *     },
+ * ]
+ * ```
+ *
+ * By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
+ * and skips pages with other content types. If you want the crawler to process other content types,
+ * use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
+ * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
+ * For more details, see {@link JSDOMCrawlerOptions.requestHandler}.
+ *
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new JSDOMCrawler({
+ *     async requestHandler({ request, window }) {
+ *         await Dataset.pushData({
+ *             url: request.url,
+ *             title: window.document.title,
+ *         });
+ *     },
+ * });
+ *
+ * await crawler.run([
+ *     'http://crawlee.dev',
+ * ]);
+ * ```
+ * @category Crawlers
+ */
+export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
     /**
      * @internal
@@ -188,7 +224,6 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
         hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strict>;
     constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, JSDOMCrawlingContext>;
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
      *
@@ -205,8 +240,6 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
      */
     getVirtualConsole(): VirtualConsole;
     private readonly jsdomErrorHandler;
-    private parseContent;
-    private addHelpers;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/jsdom-crawler.js

@@ -1,9 +1,8 @@
-import { EnqueueStrategy, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, } from '@crawlee/http';
-import { sleep } from '@crawlee/utils';
-import { parseArgument, tryAbsoluteURL } from '@crawlee/utils/internal';
-import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
+import { DOMCrawler, HttpCrawler, Router } from '@crawlee/http';
+import { parseArgument } from '@crawlee/utils/internal';
+import { VirtualConsole } from 'jsdom';
 import { z } from 'zod';
-import { addTimeoutToPromise } from '@apify/timeout';
+import { jsdomParser } from './jsdom-parser.js';
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
@@ -79,12 +78,7 @@ import { addTimeoutToPromise } from '@apify/timeout';
  * ```
  * @category Crawlers
  */
-const resources = new ResourceLoader({
-    // Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17
-    // in order not to include the entire package here
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
-});
-export class JSDOMCrawler extends HttpCrawler {
+export class JSDOMCrawler extends DOMCrawler {
     /**
      * @internal
      */
@@ -95,30 +89,21 @@ export class JSDOMCrawler extends HttpCrawler {
     };
     /** @internal */
     static optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);
-    #runScripts;
     #hideInternalConsole;
     #virtualConsole = null;
     constructor(options = {}) {
         const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');
         super({
             ...httpOptions,
-            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+            contextPipelineBuilder,
+            parser: jsdomParser({
+                runScripts,
+                virtualConsole: () => this.getVirtualConsole(),
+                log: () => this.log,
+            }),
         });
-        this.#runScripts = runScripts;
         this.#hideInternalConsole = hideInternalConsole;
     }
-    buildContextPipeline() {
-        return super
-            .buildContextPipeline()
-            .compose({
-            action: async (context) => await this.parseContent(context),
-            cleanup: async (context) => {
-                this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
-                context.window?.close();
-            },
-        })
-            .compose({ action: async (context) => await this.addHelpers(context) });
-    }
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
      *
@@ -145,141 +130,6 @@ export class JSDOMCrawler extends HttpCrawler {
         return this.#virtualConsole;
     }
     jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
-    async parseContent(crawlingContext) {
-        try {
-            const isXml = crawlingContext.contentType.type.includes('xml');
-            // TODO handle non-string
-            const { window } = new JSDOM(crawlingContext.body.toString(), {
-                url: crawlingContext.response.url,
-                contentType: isXml ? 'text/xml' : 'text/html',
-                runScripts: this.#runScripts ? 'dangerously' : undefined,
-                resources,
-                virtualConsole: this.getVirtualConsole(),
-                pretendToBeVisual: true,
-            });
-            // add some stubs in place of missing API so processing won't fail
-            Object.defineProperty(window, 'matchMedia', {
-                writable: true,
-                value: (query) => ({
-                    matches: false,
-                    media: query,
-                    onchange: null,
-                    addListener: () => { },
-                    removeListener: () => { },
-                    addEventListener: () => { },
-                    removeEventListener: () => { },
-                    dispatchEvent: () => { },
-                }),
-            });
-            window.document.createRange = () => {
-                const range = new window.Range();
-                range.getBoundingClientRect = () => ({});
-                range.getClientRects = () => ({ item: () => null, length: 0 });
-                return range;
-            };
-            if (this.#runScripts) {
-                try {
-                    await addTimeoutToPromise(async () => {
-                        return new Promise((resolve) => {
-                            window.addEventListener('load', () => {
-                                resolve();
-                            }, false);
-                        }).catch();
-                    }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
-                }
-                catch (e) {
-                    this.log.debug(e.message);
-                }
-            }
-            return {
-                window,
-                get body() {
-                    return window.document.documentElement.outerHTML;
-                },
-                get document() {
-                    return window.document;
-                },
-            };
-        }
-        catch (err) {
-            if (err instanceof NavigationSkippedError) {
-                return {
-                    get window() {
-                        throw new NavigationSkippedError('The `window` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                    get body() {
-                        throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                    get document() {
-                        throw new NavigationSkippedError('The `document` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                };
-            }
-            throw err;
-        }
-    }
-    async addHelpers(crawlingContext) {
-        const addRequests = crawlingContext.addRequests;
-        const extractLinks = async (options) => {
-            if (!crawlingContext.window) {
-                throw new Error('Cannot extract links because the JSDOM is not available.');
-            }
-            return extractUrlsFromWindow(crawlingContext.window, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
-        };
-        return {
-            extractLinks,
-            enqueueLinks: async (options = {}) => {
-                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-                    enqueueStrategy: options.strategy,
-                    finalRequestUrl: crawlingContext.request.loadedUrl,
-                    originalRequestUrl: crawlingContext.request.url,
-                    userProvidedBaseUrl: options.baseUrl,
-                });
-                const urls = await extractLinks(options);
-                return addRequests(urls, {
-                    ...options,
-                    baseUrl,
-                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
-                });
-            },
-            async waitForSelector(selector, timeoutMs = 5_000) {
-                const cheerio = await import('cheerio');
-                const $ = cheerio.load(crawlingContext.body);
-                if ($(selector).get().length === 0) {
-                    if (timeoutMs) {
-                        await sleep(50);
-                        await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
-                        return;
-                    }
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-            },
-            async parseWithCheerio(selector, _timeoutMs = 5_000) {
-                const cheerio = await import('cheerio');
-                const $ = cheerio.load(crawlingContext.body);
-                if (selector && $(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-                return $;
-            },
-        };
-    }
-}
-/**
- * Extracts URLs from a given Window object.
- * @ignore
- */
-function extractUrlsFromWindow(window, selector, baseUrl) {
-    return Array.from(window.document.querySelectorAll(selector))
-        .map((e) => e.href)
-        .filter((href) => href !== undefined && href !== '')
-        .map((href) => {
-        if (href === undefined) {
-            return undefined;
-        }
-        return tryAbsoluteURL(href, baseUrl);
-    })
-        .filter((href) => href !== undefined && href !== '');
 }
 export function createJSDOMRouter(routesOrSchemas) {
     return Router.create(routesOrSchemas);

package/internals/jsdom-parser.d.ts

@@ -0,0 +1,29 @@
+import type { DOMParser } from '@crawlee/http';
+import type { CrawleeLogger } from '@crawlee/types';
+import type { DOMWindow, VirtualConsole } from 'jsdom';
+export interface JSDOMParseResult {
+    window: DOMWindow;
+    document: Document;
+    body: string;
+}
+export interface JsdomParserOptions {
+    /**
+     * Download and run scripts.
+     */
+    runScripts?: boolean;
+    /**
+     * Resolves the `VirtualConsole` to hand to JSDOM. Called for every parse, so a crawler can create its console
+     * lazily. Defaults to JSDOM's own console handling.
+     */
+    virtualConsole?: () => VirtualConsole;
+    /**
+     * Resolves the logger for JSDOM diagnostics. Called only when there is something to log, so a crawler can hand
+     * over its own logger.
+     */
+    log?: () => CrawleeLogger;
+}
+/**
+ * A {@link DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link JSDOMCrawler} provides.
+ */
+export declare function jsdomParser(options?: JsdomParserOptions): DOMParser<JSDOMParseResult>;

package/internals/jsdom-parser.js

@@ -0,0 +1,93 @@
+import { tryAbsoluteURL } from '@crawlee/utils/internal';
+import { JSDOM, ResourceLoader } from 'jsdom';
+import { addTimeoutToPromise } from '@apify/timeout';
+const resources = new ResourceLoader({
+    // Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17
+    // in order not to include the entire package here
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
+});
+/**
+ * A {@link DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link JSDOMCrawler} provides.
+ */
+export function jsdomParser(options = {}) {
+    const { runScripts = false, virtualConsole, log } = options;
+    return {
+        placeholderMembers: { window: true, document: true, body: true },
+        mutable: runScripts,
+        async parse(context) {
+            const isXml = context.contentType.type.includes('xml');
+            // TODO handle non-string
+            const { window } = new JSDOM(context.body.toString(), {
+                url: context.response.url,
+                contentType: isXml ? 'text/xml' : 'text/html',
+                runScripts: runScripts ? 'dangerously' : undefined,
+                resources,
+                virtualConsole: virtualConsole?.(),
+                pretendToBeVisual: true,
+            });
+            // add some stubs in place of missing API so processing won't fail
+            Object.defineProperty(window, 'matchMedia', {
+                writable: true,
+                value: (query) => ({
+                    matches: false,
+                    media: query,
+                    onchange: null,
+                    addListener: () => { },
+                    removeListener: () => { },
+                    addEventListener: () => { },
+                    removeEventListener: () => { },
+                    dispatchEvent: () => { },
+                }),
+            });
+            window.document.createRange = () => {
+                const range = new window.Range();
+                range.getBoundingClientRect = () => ({});
+                range.getClientRects = () => ({ item: () => null, length: 0 });
+                return range;
+            };
+            if (runScripts) {
+                try {
+                    await addTimeoutToPromise(async () => {
+                        return new Promise((resolve) => {
+                            window.addEventListener('load', () => {
+                                resolve();
+                            }, false);
+                        }).catch();
+                    }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
+                }
+                catch (e) {
+                    log?.().debug(e.message);
+                }
+            }
+            return {
+                window,
+                get body() {
+                    return window.document.documentElement.outerHTML;
+                },
+                get document() {
+                    return window.document;
+                },
+            };
+        },
+        extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),
+        select: ({ document }, selector) => document.querySelectorAll(selector),
+        cleanup: ({ window }) => window.close(),
+    };
+}
+/**
+ * Extracts URLs from a given Window object.
+ * @ignore
+ */
+function extractUrlsFromWindow(window, selector, baseUrl) {
+    return Array.from(window.document.querySelectorAll(selector))
+        .map((e) => e.href)
+        .filter((href) => href !== undefined && href !== '')
+        .map((href) => {
+        if (href === undefined) {
+            return undefined;
+        }
+        return tryAbsoluteURL(href, baseUrl);
+    })
+        .filter((href) => href !== undefined && href !== '');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/jsdom",
-    "version": "4.0.0-beta.168",
+    "version": "4.0.0-beta.169",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,9 +49,9 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/http": "4.0.0-beta.168",
-        "@crawlee/types": "4.0.0-beta.168",
-        "@crawlee/utils": "4.0.0-beta.168",
+        "@crawlee/http": "4.0.0-beta.169",
+        "@crawlee/types": "4.0.0-beta.169",
+        "@crawlee/utils": "4.0.0-beta.169",
         "@types/jsdom": "^21.1.7",
         "cheerio": "^1.0.0",
         "jsdom": "^26.1.0",
@@ -65,5 +65,5 @@
             }
         }
     },
-    "gitHead": "fc75a7073fc330331ff2d68d81a8e6815ab4edf2"
+    "gitHead": "a42dc93fa0ef68180d83ab12af362082690f73e2"
 }