@crawlee/linkedom 4.0.0-beta.167 → 4.0.0-beta.169

package/index.d.ts

@@ -1,2 +1,3 @@
 export * from '@crawlee/http';
 export * from './internals/linkedom-crawler.js';
+export type { LinkeDOMParseResult } from './internals/linkedom-parser.js';

package/internals/linkedom-crawler.d.ts

@@ -1,7 +1,7 @@
-import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
-import { HttpCrawler } from '@crawlee/http';
+import type { CrawlingContext, DOMCrawlingContext, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
+import { DOMCrawler } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
+import type { LinkeDOMParseResult } from './linkedom-parser.js';
 export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, LinkeDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
@@ -12,44 +12,7 @@ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, Stat
 export type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;
 export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
-    window: Window;
-    document: Document;
-    /**
-     * Wait for an element matching the selector to appear.
-     * Timeout defaults to 5s.
-     *
-     * **Example usage:**
-     * ```ts
-     * async requestHandler({ waitForSelector, parseWithCheerio }) {
-     *     await waitForSelector('article h1');
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
-    /**
-     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
-     * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
-     *
-     * **Example usage:**
-     * ```javascript
-     * async requestHandler({ parseWithCheerio }) {
-     *     const $ = await parseWithCheerio();
-     *     const title = $('title').text();
-     * });
-     * ```
-     */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
-    /**
-     * Extracts URLs from the parsed DOM, without adding them to the request queue.
-     */
-    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
-    /**
-     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
-     */
-    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+JSONData extends Dictionary = any> extends DOMCrawlingContext<LinkeDOMParseResult, UserData, JSONData> {
 }
 export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
@@ -125,12 +88,8 @@ JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<User
  * ```
  * @category Crawlers
  */
-export declare class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
-    #private;
+export declare class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<LinkeDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, LinkeDOMCrawlingContext>;
-    private parseContent;
-    private addHelpers;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/linkedom-crawler.js

@@ -1,8 +1,5 @@
-import { EnqueueStrategy, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, } from '@crawlee/http';
-import { sleep } from '@crawlee/utils';
-import { tryAbsoluteURL } from '@crawlee/utils/internal';
-import * as cheerio from 'cheerio';
-import { DOMParser } from 'linkedom/cached';
+import { DOMCrawler, Router } from '@crawlee/http';
+import { linkedomParser } from './linkedom-parser.js';
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [linkedom](https://www.npmjs.com/package/linkedom) LinkeDOM implementation.
@@ -75,115 +72,10 @@ import { DOMParser } from 'linkedom/cached';
  * ```
  * @category Crawlers
  */
-export class LinkeDOMCrawler extends HttpCrawler {
-    static #parser = new DOMParser();
+export class LinkeDOMCrawler extends DOMCrawler {
     constructor(options = {}) {
-        const { contextPipelineBuilder, ...rest } = options;
-        super({
-            ...rest,
-            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
-        });
+        super({ ...options, parser: linkedomParser() });
     }
-    buildContextPipeline() {
-        return super
-            .buildContextPipeline()
-            .compose({
-            action: async (context) => this.parseContent(context),
-        })
-            .compose({ action: async (context) => this.addHelpers(context) });
-    }
-    async parseContent(crawlingContext) {
-        try {
-            const isXml = crawlingContext.contentType.type.includes('xml');
-            const document = LinkeDOMCrawler.#parser.parseFromString(crawlingContext.body.toString(), isXml ? 'text/xml' : 'text/html');
-            return {
-                window: document.defaultView,
-                get body() {
-                    return document.documentElement.outerHTML;
-                },
-                get document() {
-                    // See comment about typing in LinkeDOMCrawlingContext definition
-                    return document;
-                },
-            };
-        }
-        catch (err) {
-            if (err instanceof NavigationSkippedError) {
-                return {
-                    get window() {
-                        throw new NavigationSkippedError('The `window` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                    get body() {
-                        throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                    get document() {
-                        throw new NavigationSkippedError('The `document` property is not available - `skipNavigation` was used', { cause: err });
-                    },
-                };
-            }
-            throw err;
-        }
-    }
-    async addHelpers(crawlingContext) {
-        const addRequests = crawlingContext.addRequests;
-        const extractLinks = async (options) => {
-            if (!crawlingContext.window) {
-                throw new Error('Cannot extract links because the DOM is not available.');
-            }
-            return extractUrlsFromWindow(crawlingContext.window, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
-        };
-        return {
-            extractLinks,
-            enqueueLinks: async (options = {}) => {
-                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-                    enqueueStrategy: options.strategy,
-                    finalRequestUrl: crawlingContext.request.loadedUrl,
-                    originalRequestUrl: crawlingContext.request.url,
-                    userProvidedBaseUrl: options.baseUrl,
-                });
-                const urls = await extractLinks(options);
-                return addRequests(urls, {
-                    ...options,
-                    baseUrl,
-                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
-                });
-            },
-            async waitForSelector(selector, timeoutMs = 5_000) {
-                const $ = cheerio.load(crawlingContext.body);
-                if ($(selector).get().length === 0) {
-                    if (timeoutMs) {
-                        await sleep(50);
-                        await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
-                        return;
-                    }
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-            },
-            async parseWithCheerio(selector, _timeoutMs = 5_000) {
-                const $ = cheerio.load(crawlingContext.body);
-                if (selector && $(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-                return $;
-            },
-        };
-    }
-}
-/**
- * Extracts URLs from a given Window object.
- * @ignore
- */
-function extractUrlsFromWindow(window, selector, baseUrl) {
-    return Array.from(window.document.querySelectorAll(selector))
-        .map((e) => e.href)
-        .filter((href) => href !== undefined && href !== '')
-        .map((href) => {
-        if (href === undefined) {
-            return undefined;
-        }
-        return tryAbsoluteURL(href, baseUrl);
-    })
-        .filter((href) => href !== undefined && href !== '');
 }
 export function createLinkeDOMRouter(routesOrSchemas) {
     return Router.create(routesOrSchemas);

package/internals/linkedom-parser.d.ts

@@ -0,0 +1,11 @@
+import type { DOMParser } from '@crawlee/http';
+export interface LinkeDOMParseResult {
+    window: Window;
+    document: Document;
+    body: string;
+}
+/**
+ * A {@link DOMParser} backed by [linkedom](https://www.npmjs.com/package/linkedom). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link LinkeDOMCrawler} provides.
+ */
+export declare function linkedomParser(): DOMParser<LinkeDOMParseResult>;

package/internals/linkedom-parser.js

@@ -0,0 +1,44 @@
+import { tryAbsoluteURL } from '@crawlee/utils/internal';
+import { DOMParser as LinkeDOMParser } from 'linkedom/cached';
+/**
+ * A {@link DOMParser} backed by [linkedom](https://www.npmjs.com/package/linkedom). Pass it to a
+ * {@link DOMCrawler} to get the crawling context {@link LinkeDOMCrawler} provides.
+ */
+export function linkedomParser() {
+    const parser = new LinkeDOMParser();
+    return {
+        placeholderMembers: { window: true, document: true, body: true },
+        parse(context) {
+            const isXml = context.contentType.type.includes('xml');
+            const document = parser.parseFromString(context.body.toString(), isXml ? 'text/xml' : 'text/html');
+            return {
+                window: document.defaultView,
+                get body() {
+                    return document.documentElement.outerHTML;
+                },
+                get document() {
+                    // See comment about typing in LinkeDOMParseResult definition
+                    return document;
+                },
+            };
+        },
+        extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),
+        select: ({ document }, selector) => document.querySelectorAll(selector),
+    };
+}
+/**
+ * Extracts URLs from a given Window object.
+ * @ignore
+ */
+function extractUrlsFromWindow(window, selector, baseUrl) {
+    return Array.from(window.document.querySelectorAll(selector))
+        .map((e) => e.href)
+        .filter((href) => href !== undefined && href !== '')
+        .map((href) => {
+        if (href === undefined) {
+            return undefined;
+        }
+        return tryAbsoluteURL(href, baseUrl);
+    })
+        .filter((href) => href !== undefined && href !== '');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/linkedom",
-    "version": "4.0.0-beta.167",
+    "version": "4.0.0-beta.169",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,9 +49,9 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/http": "4.0.0-beta.167",
-        "@crawlee/types": "4.0.0-beta.167",
-        "@crawlee/utils": "4.0.0-beta.167",
+        "@crawlee/http": "4.0.0-beta.169",
+        "@crawlee/types": "4.0.0-beta.169",
+        "@crawlee/utils": "4.0.0-beta.169",
         "cheerio": "^1.0.0",
         "linkedom": "^0.18.10",
         "tslib": "^2.8.1"
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "8b26080bf07dc602aa0970e12fb66e7e9cba509b"
+    "gitHead": "a42dc93fa0ef68180d83ab12af362082690f73e2"
 }