@crawlee/linkedom 4.0.0-beta.99 → 4.0.0-rc.0

package/internals/linkedom-crawler.d.ts

@@ -1,16 +1,14 @@
-import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
+import type { AddRequestsBatchedResult, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
 import { HttpCrawler } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
-import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import * as cheerio from 'cheerio';
 export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, LinkeDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
 export interface LinkeDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
-}
-export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {
+Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
 }
 export type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;
@@ -45,6 +43,14 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
      * ```
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    /**
+     * Extracts URLs from the parsed DOM, without adding them to the request queue.
+     */
+    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
+    /**
+     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
+     */
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
 }
 export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
@@ -120,40 +126,23 @@ JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<User
  * ```
  * @category Crawlers
  */
-export declare class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
-    private static parser;
-    constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
+export declare class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
         readonly window: Window;
         readonly body: string;
         readonly document: Document;
     } & {
-        enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<unknown>;
+        extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
+        enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
         waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
         parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
     }>;
     private parseContent;
     private addHelpers;
 }
-interface EnqueueLinksInternalOptions {
-    options?: EnqueueLinksOptions;
-    window: Window | null;
-    requestManager: IRequestManager;
-    robotsTxtFile?: RobotsTxtFile;
-    onSkippedRequest?: SkippedRequestCallback;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-}
-interface BoundEnqueueLinksInternalOptions {
-    enqueueLinks: BasicCrawlingContext['enqueueLinks'];
-    options?: EnqueueLinksOptions;
-    window: Window | null;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-}
-/** @internal */
-export declare function linkedomCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link LinkeDOMCrawler}.
@@ -181,4 +170,3 @@ export declare function linkedomCrawlerEnqueueLinks(options: EnqueueLinksInterna
 export declare function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 export declare function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
-export {};

package/internals/linkedom-crawler.js

@@ -1,4 +1,4 @@
-import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
+import { EnqueueStrategy, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
 import { sleep } from '@crawlee/utils';
 import * as cheerio from 'cheerio';
 import { DOMParser } from 'linkedom/cached';
@@ -75,8 +75,8 @@ import { DOMParser } from 'linkedom/cached';
  * @category Crawlers
  */
 export class LinkeDOMCrawler extends HttpCrawler {
-    static parser = new DOMParser();
-    constructor(options) {
+    static #parser = new DOMParser();
+    constructor(options = {}) {
         const { contextPipelineBuilder, ...rest } = options;
         super({
             ...rest,
@@ -94,7 +94,7 @@ export class LinkeDOMCrawler extends HttpCrawler {
     async parseContent(crawlingContext) {
         try {
             const isXml = crawlingContext.contentType.type.includes('xml');
-            const document = LinkeDOMCrawler.parser.parseFromString(crawlingContext.body.toString(), isXml ? 'text/xml' : 'text/html');
+            const document = LinkeDOMCrawler.#parser.parseFromString(crawlingContext.body.toString(), isXml ? 'text/xml' : 'text/html');
             return {
                 window: document.defaultView,
                 get body() {
@@ -124,19 +124,27 @@ export class LinkeDOMCrawler extends HttpCrawler {
         }
     }
     async addHelpers(crawlingContext) {
+        const addRequests = crawlingContext.addRequests;
+        const extractLinks = async (options) => {
+            if (!crawlingContext.window) {
+                throw new Error('Cannot extract links because the DOM is not available.');
+            }
+            return extractUrlsFromWindow(crawlingContext.window, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
+        };
         return {
-            enqueueLinks: async (enqueueOptions) => {
-                return linkedomCrawlerEnqueueLinks({
-                    options: {
-                        ...enqueueOptions,
-                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
-                    },
-                    window: document.defaultView,
-                    requestManager: await this.getRequestManager(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.handleSkippedRequest,
-                    originalRequestUrl: crawlingContext.request.url,
+            extractLinks,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
+                    originalRequestUrl: crawlingContext.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
                 });
             },
             async waitForSelector(selector, timeoutMs = 5_000) {
@@ -160,39 +168,6 @@ export class LinkeDOMCrawler extends HttpCrawler {
         };
     }
 }
-/** @internal */
-function containsEnqueueLinks(options) {
-    return !!options.enqueueLinks;
-}
-/** @internal */
-export async function linkedomCrawlerEnqueueLinks(options) {
-    const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
-    if (!window) {
-        throw new Error('Cannot enqueue links because the DOM is not available.');
-    }
-    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: enqueueLinksOptions?.strategy,
-        finalRequestUrl,
-        originalRequestUrl,
-        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
-    });
-    const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    if (containsEnqueueLinks(options)) {
-        return options.enqueueLinks({
-            urls,
-            baseUrl,
-            ...enqueueLinksOptions,
-        });
-    }
-    return enqueueLinks({
-        requestManager: options.requestManager,
-        robotsTxtFile: options.robotsTxtFile,
-        onSkippedRequest: options.onSkippedRequest,
-        urls,
-        baseUrl,
-        ...enqueueLinksOptions,
-    });
-}
 /**
  * Extracts URLs from a given Window object.
  * @ignore

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/linkedom",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,12 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/http": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
+        "@crawlee/http": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/utils": "4.0.0-rc.0",
         "cheerio": "^1.0.0",
         "linkedom": "^0.18.10",
-        "ow": "^2.0.0",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -64,5 +63,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }