@crawlee/jsdom 4.0.0-beta.7 → 4.0.0-beta.70

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/internals/jsdom-crawler.d.ts

@@ -1,5 +1,4 @@
-import type { IncomingMessage } from 'node:http';
-import type { Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
+import type { BasicCrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
 import { HttpCrawler } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
 import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
@@ -7,8 +6,8 @@ import type { DOMWindow } from 'jsdom';
 import { VirtualConsole } from 'jsdom';
 export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
-export interface JSDOMCrawlerOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>> {
+export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
     /**
      * Download and run scripts.
      */
@@ -21,9 +20,10 @@ JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingConte
 export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
 export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {
+JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
     window: DOMWindow;
     document: Document;
+    body: string;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -54,7 +54,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
 }
 export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
-export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
+export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
@@ -71,17 +71,15 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
 // @ts-ignore optional peer dependency or compatibility with es2022
         forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -98,28 +96,42 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -128,17 +140,26 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     protected runScripts: boolean;
     protected hideInternalConsole: boolean;
     protected virtualConsole: VirtualConsole | null;
-    constructor(options?: JSDOMCrawlerOptions, config?: Configuration);
+    constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
+// @ts-ignore optional peer dependency or compatibility with es2022
+    protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<import("@crawlee/http").CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
+        readonly window: DOMWindow;
+        readonly body: string;
+        readonly document: Document;
+    } & {
+        enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
+        waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
+    }>;
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
      *
@@ -155,28 +176,27 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
      */
     getVirtualConsole(): VirtualConsole;
     private readonly jsdomErrorHandler;
-    protected _cleanupContext(context: JSDOMCrawlingContext): Promise<void>;
-    protected _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: JSDOMCrawlingContext): Promise<{
-        window: DOMWindow;
-        readonly body: string;
-        readonly document: Document;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
-    }>;
-    _runRequestHandler(context: JSDOMCrawlingContext): Promise<void>;
+    private parseContent;
+    private addHelpers;
 }
 interface EnqueueLinksInternalOptions {
     options?: EnqueueLinksOptions;
     window: DOMWindow | null;
-    requestQueue: RequestProvider;
+    requestManager: IRequestManager;
     robotsTxtFile?: RobotsTxtFile;
     onSkippedRequest?: SkippedRequestCallback;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
+interface BoundEnqueueLinksInternalOptions {
+    enqueueLinks: BasicCrawlingContext['enqueueLinks'];
+    options?: EnqueueLinksOptions;
+    window: DOMWindow | null;
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
+}
 /** @internal */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
+export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.

package/internals/jsdom-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsdom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EACR,aAAa,EACb,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,sBAAsB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,WAAW,EAId,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAS,MAAM,gBAAgB,CAAC;AAE7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAyB,cAAc,EAAE,MAAM,OAAO,CAAC;AAM9D,MAAM,MAAM,iBAAiB,CACzB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,WAAW,mBAAmB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClE;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,MAAM,SAAS,CACjB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,oBAAoB,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC;IACnE,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAkF7D,qBAAa,YAAa,SAAQ,WAAW,CAAC,oBAAoB,CAAC;IAC/D,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAQ;gBAE3C,OAAO,GAAE,mBAAwB,EAAE,MAAM,CAAC,EAAE,aAAa;IASrE;;;;;;;;;;;;;OAaG;IACH,iBAAiB;IAgBjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAuE;cAEhF,eAAe,CAAC,OAAO,EAAE,oBAAoB;cAK7C,UAAU,CAC/B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,OAAO,EACd,eAAe,EAAE,oBAAoB;;;;wCAgEK,mBAAmB;;IAclD,kBAAkB,CAAC,OAAO,EAAE,oBAAoB;CA0BlE;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,sBAAsB,CAAC,EACzC,OAAO,EACP,MAAM,EACN,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,4DA0B7B;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAC7B,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}
+{"version":3,"file":"jsdom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,oBAAoB,EACpB,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,YAAY,EACZ,sBAAsB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,WAAW,EAKd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAS,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAyB,cAAc,EAAE,MAAM,OAAO,CAAC;AAK9D,MAAM,MAAM,iBAAiB,CACzB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,WAAW,mBAAmB,CAChC,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,GAAG,gBAAgB,EACtF,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrG;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,MAAM,SAAS,CACjB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,oBAAoB,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACrD,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IAEnB,IAAI,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAoF7D,qBAAa,YAAY,CACrB,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,GAAG,gBAAgB,CACxF,SAAQ,WAAW,CAAC,oBAAoB,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC1E,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAQ;gBAE3C,OAAO,GAAE,mBAAmB,CAAC,gBAAgB,EAAE,eAAe,CAAM;cAY7D,oBAAoB;yBAkHT,SAAS;uBAMX,MAAM;2BAMF,QAAQ;;wCAeM,mBAAmB;kCAczB,MAAM;oCAcJ,MAAM;;IA5JhD;;;;;;;;;;;;;OAaG;IACH,iBAAiB;IAgBjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA2E;YAE/F,YAAY;YA8FZ,UAAU;CA0C3B;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,cAAc,EAAE,eAAe,CAAC;IAChC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,UAAU,gCAAgC;IACtC,YAAY,EAAE,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AASD,gBAAgB;AAChB,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,gCAAgC,oBAoCnH;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAC7B,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}

package/internals/jsdom-crawler.js

@@ -1,10 +1,8 @@
-import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
+import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
 import { sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
 import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
 import ow from 'ow';
 import { addTimeoutToPromise } from '@apify/timeout';
-import { concatStreamToBuffer } from '@apify/utilities';
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
@@ -27,28 +25,30 @@ import { concatStreamToBuffer } from '@apify/utilities';
  * and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
  * using the `window` object.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link JSDOMCrawlerOptions.requestList}
- * or {@link JSDOMCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link JSDOMCrawlerOptions.requestList} and {@link JSDOMCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, `JSDOMCrawler` only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
@@ -92,12 +92,27 @@ export class JSDOMCrawler extends HttpCrawler {
     runScripts;
     hideInternalConsole;
     virtualConsole = null;
-    constructor(options = {}, config) {
-        const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
-        super(httpOptions, config);
+    constructor(options = {}) {
+        const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
+        super({
+            ...httpOptions,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
         this.runScripts = runScripts;
         this.hideInternalConsole = hideInternalConsole;
     }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose({
+            action: async (context) => await this.parseContent(context),
+            cleanup: async (context) => {
+                this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
+                context.window?.close();
+            },
+        })
+            .compose({ action: async (context) => await this.addHelpers(context) });
+    }
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
      *
@@ -123,117 +138,150 @@ export class JSDOMCrawler extends HttpCrawler {
         this.virtualConsole.on('jsdomError', this.jsdomErrorHandler);
         return this.virtualConsole;
     }
-    jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', error);
-    async _cleanupContext(context) {
-        this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
-        context.window?.close();
-    }
-    async _parseHTML(response, isXml, crawlingContext) {
-        const body = await concatStreamToBuffer(response);
-        const { window } = new JSDOM(body, {
-            url: response.url,
-            contentType: isXml ? 'text/xml' : 'text/html',
-            runScripts: this.runScripts ? 'dangerously' : undefined,
-            resources,
-            virtualConsole: this.getVirtualConsole(),
-            pretendToBeVisual: true,
-        });
-        // add some stubs in place of missing API so processing won't fail
-        Object.defineProperty(window, 'matchMedia', {
-            writable: true,
-            value: (query) => ({
-                matches: false,
-                media: query,
-                onchange: null,
-                addListener: () => { },
-                removeListener: () => { },
-                addEventListener: () => { },
-                removeEventListener: () => { },
-                dispatchEvent: () => { },
-            }),
-        });
-        window.document.createRange = () => {
-            const range = new window.Range();
-            range.getBoundingClientRect = () => ({});
-            range.getClientRects = () => ({ item: () => null, length: 0 });
-            return range;
-        };
-        if (this.runScripts) {
-            try {
-                await addTimeoutToPromise(async () => {
-                    return new Promise((resolve) => {
-                        window.addEventListener('load', () => {
-                            resolve();
-                        }, false);
-                    }).catch();
-                }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
+    jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
+    async parseContent(crawlingContext) {
+        try {
+            const isXml = crawlingContext.contentType.type.includes('xml');
+            // TODO handle non-string
+            const { window } = new JSDOM(crawlingContext.body.toString(), {
+                url: crawlingContext.response.url,
+                contentType: isXml ? 'text/xml' : 'text/html',
+                runScripts: this.runScripts ? 'dangerously' : undefined,
+                resources,
+                virtualConsole: this.getVirtualConsole(),
+                pretendToBeVisual: true,
+            });
+            // add some stubs in place of missing API so processing won't fail
+            Object.defineProperty(window, 'matchMedia', {
+                writable: true,
+                value: (query) => ({
+                    matches: false,
+                    media: query,
+                    onchange: null,
+                    addListener: () => { },
+                    removeListener: () => { },
+                    addEventListener: () => { },
+                    removeEventListener: () => { },
+                    dispatchEvent: () => { },
+                }),
+            });
+            window.document.createRange = () => {
+                const range = new window.Range();
+                range.getBoundingClientRect = () => ({});
+                range.getClientRects = () => ({ item: () => null, length: 0 });
+                return range;
+            };
+            if (this.runScripts) {
+                try {
+                    await addTimeoutToPromise(async () => {
+                        return new Promise((resolve) => {
+                            window.addEventListener('load', () => {
+                                resolve();
+                            }, false);
+                        }).catch();
+                    }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
+                }
+                catch (e) {
+                    this.log.debug(e.message);
+                }
             }
-            catch (e) {
-                this.log.debug(e.message);
+            return {
+                window,
+                get body() {
+                    return window.document.documentElement.outerHTML;
+                },
+                get document() {
+                    return window.document;
+                },
+            };
+        }
+        catch (err) {
+            if (err instanceof NavigationSkippedError) {
+                return {
+                    get window() {
+                        throw new NavigationSkippedError('The `window` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                    get body() {
+                        throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                    get document() {
+                        throw new NavigationSkippedError('The `document` property is not available - `skipNavigation` was used', { cause: err });
+                    },
+                };
             }
+            throw err;
         }
+    }
+    async addHelpers(crawlingContext) {
         return {
-            window,
-            get body() {
-                return window.document.documentElement.outerHTML;
-            },
-            get document() {
-                return window.document;
-            },
             enqueueLinks: async (enqueueOptions) => {
                 return domCrawlerEnqueueLinks({
-                    options: enqueueOptions,
-                    window,
-                    requestQueue: await this.getRequestQueue(),
+                    options: {
+                        ...enqueueOptions,
+                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
+                    },
+                    window: crawlingContext.window,
+                    requestManager: await this.getRequestManager(),
                     robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.onSkippedRequest,
+                    onSkippedRequest: this.handleSkippedRequest,
                     originalRequestUrl: crawlingContext.request.url,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
                 });
             },
-        };
-    }
-    async _runRequestHandler(context) {
-        context.waitForSelector = async (selector, timeoutMs = 5_000) => {
-            const $ = cheerio.load(context.body);
-            if ($(selector).get().length === 0) {
-                if (timeoutMs) {
-                    await sleep(50);
-                    await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
-                    return;
+            async waitForSelector(selector, timeoutMs = 5_000) {
+                const cheerio = await import('cheerio');
+                const $ = cheerio.load(crawlingContext.body);
+                if ($(selector).get().length === 0) {
+                    if (timeoutMs) {
+                        await sleep(50);
+                        await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
+                        return;
+                    }
+                    throw new Error(`Selector '${selector}' not found.`);
                 }
-                throw new Error(`Selector '${selector}' not found.`);
-            }
-        };
-        context.parseWithCheerio = async (selector, _timeoutMs = 5_000) => {
-            const $ = cheerio.load(context.body);
-            if (selector && $(selector).get().length === 0) {
-                throw new Error(`Selector '${selector}' not found.`);
-            }
-            return $;
+            },
+            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                const cheerio = await import('cheerio');
+                const $ = cheerio.load(crawlingContext.body);
+                if (selector && $(selector).get().length === 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                return $;
+            },
         };
-        await super._runRequestHandler(context);
     }
 }
 /** @internal */
-export async function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
+function containsEnqueueLinks(options) {
+    return !!options.enqueueLinks;
+}
+/** @internal */
+export async function domCrawlerEnqueueLinks(options) {
+    const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
     if (!window) {
         throw new Error('Cannot enqueue links because the JSDOM is not available.');
     }
     const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: options?.strategy,
+        enqueueStrategy: enqueueLinksOptions?.strategy,
         finalRequestUrl,
         originalRequestUrl,
-        userProvidedBaseUrl: options?.baseUrl,
+        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
     });
-    const urls = extractUrlsFromWindow(window, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    if (containsEnqueueLinks(options)) {
+        return options.enqueueLinks({
+            urls,
+            baseUrl,
+            ...enqueueLinksOptions,
+        });
+    }
     return enqueueLinks({
-        requestQueue,
-        robotsTxtFile,
-        onSkippedRequest,
+        requestManager: options.requestManager,
+        robotsTxtFile: options.robotsTxtFile,
+        onSkippedRequest: options.onSkippedRequest,
         urls,
         baseUrl,
-        ...options,
+        ...enqueueLinksOptions,
     });
 }
 /**