@crawlee/cheerio 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,2 +1,3 @@
 export * from '@crawlee/http';
 export * from './internals/cheerio-crawler.js';
+export type { CheerioParseResult } from './internals/cheerio-parser.js';

package/internals/cheerio-crawler.d.ts

@@ -1,61 +1,16 @@
-import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
-import { HttpCrawler } from '@crawlee/http';
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
-import type { CheerioAPI } from 'cheerio';
-import * as cheerio from 'cheerio';
+import type { CrawlingContext, DOMCrawlingContext, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
+import { DOMCrawler } from '@crawlee/http';
+import type { Dictionary } from '@crawlee/types';
+import type { CheerioParseResult } from './cheerio-parser.js';
 export type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, CheerioCrawlingContext<UserData, JSONData> & ContextExtension>;
 export interface CheerioCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
+Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
 }
-export type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
 export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
-    /**
-     * The raw HTML content of the web page as a string.
-     */
-    body: string;
-    /**
-     * The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
-     * Cheerio is available only for HTML and XML content types.
-     */
-    $: cheerio.CheerioAPI;
-    /**
-     * Wait for an element matching the selector to appear. Timeout is ignored.
-     *
-     * **Example usage:**
-     * ```ts
-     * async requestHandler({ waitForSelector, parseWithCheerio }) {
-     *     await waitForSelector('article h1');
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-    /**
-     * Returns Cheerio handle, this is here to unify the crawler API, so they all have this handy method.
-     * It has the same return type as the `$` context property, use it only if you are abstracting your workflow to
-     * support different context types in one handler.
-     * When provided with the `selector` argument, it will throw if it's not available.
-     *
-     * **Example usage:**
-     * ```ts
-     * async requestHandler({ parseWithCheerio }) {
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
-    /**
-     * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
-     */
-    enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
+JSONData extends Dictionary = any> extends DOMCrawlingContext<CheerioParseResult, UserData, JSONData> {
 }
 export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
@@ -138,41 +93,12 @@ JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserD
  * ```
  * @category Crawlers
  */
-export declare class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+export declare class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<CheerioParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
-        readonly body: string;
-        readonly $: CheerioAPI;
-    } & {
-        enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
-        waitForSelector: (selector: string, _timeoutMs?: number) => Promise<void>;
-        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
-    }>;
-    private parseContent;
-    private addHelpers;
-}
-interface EnqueueLinksInternalOptions {
-    options?: EnqueueLinksOptions;
-    $: cheerio.CheerioAPI | null;
-    requestManager: IRequestManager;
-    robotsTxtFile?: RobotsTxtFile;
-    onSkippedRequest?: SkippedRequestCallback;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-}
-interface BoundEnqueueLinksInternalOptions {
-    enqueueLinks: BasicCrawlingContext['enqueueLinks'];
-    options?: EnqueueLinksOptions;
-    $: cheerio.CheerioAPI | null;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
+    constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
 }
-/** @internal */
-export declare function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
@@ -200,4 +126,3 @@ export declare function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternal
 export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
-export {};

package/internals/cheerio-crawler.js

@@ -1,7 +1,5 @@
-import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, } from '@crawlee/http';
-import { extractUrlsFromCheerio } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import { parseDocument } from 'htmlparser2';
+import { DOMCrawler, Router } from '@crawlee/http';
+import { cheerioParser } from './cheerio-parser.js';
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
@@ -81,118 +79,13 @@ import { parseDocument } from 'htmlparser2';
  * ```
  * @category Crawlers
  */
-export class CheerioCrawler extends HttpCrawler {
+export class CheerioCrawler extends DOMCrawler {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
     constructor(options) {
-        const { contextPipelineBuilder, ...rest } = options ?? {};
-        super({
-            ...rest,
-            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
-        });
+        super({ ...options, parser: cheerioParser() });
     }
-    buildContextPipeline() {
-        return super
-            .buildContextPipeline()
-            .compose({
-            action: async (context) => await this.parseContent(context),
-        })
-            .compose({ action: async (context) => await this.addHelpers(context) });
-    }
-    async parseContent(crawlingContext) {
-        try {
-            const isXml = crawlingContext.contentType.type.includes('xml');
-            const body = Buffer.isBuffer(crawlingContext.body)
-                ? crawlingContext.body.toString(crawlingContext.contentType.encoding)
-                : crawlingContext.body;
-            const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
-            const $ = cheerio.load(dom, {
-                xml: { decodeEntities: true, xmlMode: isXml },
-            });
-            return {
-                $,
-                body,
-            };
-        }
-        catch (err) {
-            if (err instanceof NavigationSkippedError) {
-                return {
-                    get body() {
-                        throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                    get $() {
-                        throw new NavigationSkippedError('The `$` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                };
-            }
-            throw err;
-        }
-    }
-    async addHelpers(crawlingContext) {
-        const originalEnqueueLinks = crawlingContext.enqueueLinks;
-        return {
-            enqueueLinks: async (enqueueOptions) => {
-                return (await cheerioCrawlerEnqueueLinks({
-                    options: {
-                        ...enqueueOptions,
-                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
-                    },
-                    $: crawlingContext.$,
-                    requestManager: await this.getRequestManager(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.handleSkippedRequest,
-                    originalRequestUrl: crawlingContext.request.url,
-                    finalRequestUrl: crawlingContext.request.loadedUrl,
-                    enqueueLinks: originalEnqueueLinks,
-                })); // TODO make this type safe
-            },
-            waitForSelector: async (selector, _timeoutMs) => {
-                if (crawlingContext.$(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-            },
-            parseWithCheerio: async (selector, timeoutMs) => {
-                if (selector) {
-                    await crawlingContext.waitForSelector(selector, timeoutMs);
-                }
-                return crawlingContext.$;
-            },
-        };
-    }
-}
-/** @internal */
-function containsEnqueueLinks(options) {
-    return !!options.enqueueLinks;
-}
-/** @internal */
-export async function cheerioCrawlerEnqueueLinks(options) {
-    const { options: enqueueLinksOptions, $, originalRequestUrl, finalRequestUrl } = options;
-    if (!$) {
-        throw new Error('Cannot enqueue links because the DOM is not available.');
-    }
-    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: enqueueLinksOptions?.strategy,
-        finalRequestUrl,
-        originalRequestUrl,
-        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
-    });
-    const urls = extractUrlsFromCheerio($, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    if (containsEnqueueLinks(options)) {
-        return options.enqueueLinks({
-            urls,
-            baseUrl,
-            ...enqueueLinksOptions,
-        });
-    }
-    return enqueueLinks({
-        requestManager: options.requestManager,
-        robotsTxtFile: options.robotsTxtFile,
-        onSkippedRequest: options.onSkippedRequest,
-        urls,
-        baseUrl,
-        ...enqueueLinksOptions,
-    });
 }
 export function createCheerioRouter(routesOrSchemas) {
     return Router.create(routesOrSchemas);

package/internals/cheerio-parser.d.ts

@@ -0,0 +1,11 @@
+import type { DOMParser } from '@crawlee/http';
+import type { CheerioAPI } from 'cheerio';
+export interface CheerioParseResult {
+    $: CheerioAPI;
+    body: string;
+}
+/**
+ * A {@link DOMParser} backed by [cheerio](https://www.npmjs.com/package/cheerio). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link CheerioCrawler} provides.
+ */
+export declare function cheerioParser(): DOMParser<CheerioParseResult>;

package/internals/cheerio-parser.js

@@ -0,0 +1,27 @@
+import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
+// slim uses faster htmlparser2 parser and doesn't load extra libs like undici
+import * as cheerio from 'cheerio/slim';
+import { parseDocument } from 'htmlparser2';
+/**
+ * A {@link DOMParser} backed by [cheerio](https://www.npmjs.com/package/cheerio). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link CheerioCrawler} provides.
+ */
+export function cheerioParser() {
+    return {
+        placeholderMembers: { $: true, body: true },
+        parse(context) {
+            const isXml = context.contentType.type.includes('xml');
+            const body = Buffer.isBuffer(context.body)
+                ? context.body.toString(context.contentType.encoding)
+                : context.body;
+            const dom = parseDocument(body, { decodeEntities: true, xmlMode: isXml });
+            const $ = cheerio.load(dom, {
+                xml: { decodeEntities: true, xmlMode: isXml },
+            });
+            return { $, body };
+        },
+        extractLinks: ({ $ }, selector, baseUrl) => extractUrlsFromCheerio($, selector, baseUrl),
+        select: ({ $ }, selector) => $(selector).get(),
+        toCheerio: ({ $ }) => $,
+    };
+}

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/cheerio",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,9 +47,9 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/http": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
+        "@crawlee/http": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "cheerio": "^1.0.0",
         "htmlparser2": "^10.0.0",
         "tslib": "^2.8.1"
@@ -61,5 +61,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }