@crawlee/http 4.0.0-beta.230 → 4.0.0-beta.232

package/internals/dom-crawler.d.ts

@@ -31,7 +31,7 @@ export interface DOMParseResult {
  * });
  * ```
  */
-export interface DOMParser<Parsed extends DOMParseResult> {
+export interface DOMParser<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
     /**
      * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
      * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
@@ -45,10 +45,15 @@ export interface DOMParser<Parsed extends DOMParseResult> {
      */
     extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
     /**
-     * Returns the current matches of `selector`. Only the count is used, by
-     * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+     * Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
+     * {@link DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
      */
-    select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+    querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
+    /**
+     * Returns the first match of `selector`, or `undefined` when there is none - this is what
+     * {@link DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
+     */
+    querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
     /**
      * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
      * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
@@ -67,7 +72,7 @@ export interface DOMParser<Parsed extends DOMParseResult> {
      */
     cleanup?(parsed: Parsed): Awaitable<void>;
 }
-export interface DOMCrawlingHelpers {
+export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
     /**
      * Extracts URLs from the parsed DOM, without adding them to the request queue.
      */
@@ -92,6 +97,15 @@ export interface DOMCrawlingHelpers {
      * ```
      */
     waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Like {@link DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
+     * the parser produces them - a Cheerio object in {@link CheerioCrawler}. Throws when nothing matches.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
+    /**
+     * Like {@link DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
+     */
+    querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
     /**
      * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
      * When provided with the `selector` argument, it will throw if it's not available.
@@ -106,14 +120,14 @@ export interface DOMCrawlingHelpers {
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
 }
-export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
-export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
     /**
      * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
      * context, so the members the request handler receives follow from the parser you pass.
      */
-    parser: DOMParser<Parsed>;
+    parser: DOMParser<Parsed, Selection, Single>;
 }
 /**
  * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
@@ -124,8 +138,8 @@ export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResul
  *
  * @category Crawlers
  */
-export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
-    constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+    constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
 }

package/internals/dom-crawler.js

@@ -50,19 +50,29 @@ export class DOMCrawler extends HttpCrawler {
         const { addRequests } = context;
         const parser = this.#parser;
         const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
-        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+        // Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
+        const poll = async (selector, timeoutMs, query, isEmpty) => {
             let remaining = parser.mutable ? timeoutMs : 0;
-            while ((await parser.select(context, selector)).length === 0) {
+            let result = await query();
+            while (isEmpty(result)) {
                 if (remaining <= 0) {
                     throw new Error(`Selector '${selector}' not found.`);
                 }
                 await sleep(50);
                 remaining -= 50;
+                result = await query();
             }
+            return result;
+        };
+        const querySelectorAll = async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorAll(context, selector), (matches) => matches.length === 0);
+        const waitForSelector = async (selector, timeoutMs) => {
+            await querySelectorAll(selector, timeoutMs);
         };
         return {
             extractLinks,
             waitForSelector,
+            querySelectorAll,
+            querySelectorOne: async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorOne(context, selector), (match) => match === undefined),
             enqueueLinks: async (options = {}) => {
                 const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
                     enqueueStrategy: options.strategy,
@@ -77,16 +87,15 @@ export class DOMCrawler extends HttpCrawler {
                     strategy: options.strategy ?? EnqueueStrategy.SameHostname,
                 });
             },
-            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+            async parseWithCheerio(selector, timeoutMs) {
+                if (selector) {
+                    await waitForSelector(selector, timeoutMs);
+                }
                 // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
-                const $ = (await parser.toCheerio?.(context)) ??
+                return ((await parser.toCheerio?.(context)) ??
                     (await import('cheerio')).load(context.body, {
                         xmlMode: context.contentType.type.includes('xml'),
-                    });
-                if (selector && $(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-                return $;
+                    }));
             },
         };
     }

package/internals/http-crawler.d.ts

@@ -1,7 +1,8 @@
 import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
 import type { JsonValue } from 'type-fest';
 import { z } from 'zod';
 /**
@@ -162,6 +163,17 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
      * ```
      */
     waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
+     * elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
+     * matches.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
+    /**
+     * Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
+     * match.
+     */
+    querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
     /**
      * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
      * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
@@ -182,6 +194,8 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
 }
 export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
 }
 export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;

package/internals/http-crawler.js

@@ -258,6 +258,12 @@ export class HttpCrawler extends BasicCrawler {
                 get parseWithCheerio() {
                     throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
                 },
+                get querySelectorAll() {
+                    throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
+                },
+                get querySelectorOne() {
+                    throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
+                },
             };
         }
         tryCancel();
@@ -287,11 +293,16 @@ export class HttpCrawler extends BasicCrawler {
             const { load } = await import('cheerio/slim');
             return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
         };
-        const waitForSelector = async (selector, _timeoutMs) => {
-            const $ = await loadBody();
-            if ($(selector).get().length === 0) {
+        const querySelectorAll = async (selector, _timeoutMs) => {
+            const matches = (await loadBody())(selector);
+            if (matches.length === 0) {
                 throw new Error(`Selector '${selector}' not found.`);
             }
+            return matches;
+        };
+        const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
+        const waitForSelector = async (selector, timeoutMs) => {
+            await querySelectorAll(selector, timeoutMs);
         };
         const parseWithCheerio = async (selector, timeoutMs) => {
             const $ = await loadBody();
@@ -328,6 +339,8 @@ export class HttpCrawler extends BasicCrawler {
                 return JSON.parse(jsonString);
             },
             waitForSelector,
+            querySelectorOne,
+            querySelectorAll,
             parseWithCheerio,
             contentType,
             body: parsed.body,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.230",
+    "version": "4.0.0-beta.232",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -48,12 +48,13 @@
     },
     "dependencies": {
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.230",
-        "@crawlee/http-client": "4.0.0-beta.230",
-        "@crawlee/types": "4.0.0-beta.230",
-        "@crawlee/utils": "4.0.0-beta.230",
+        "@crawlee/basic": "4.0.0-beta.232",
+        "@crawlee/http-client": "4.0.0-beta.232",
+        "@crawlee/types": "4.0.0-beta.232",
+        "@crawlee/utils": "4.0.0-beta.232",
         "cheerio": "^1.0.0",
         "content-type": "^3.1.1",
+        "domhandler": "^6.0.0",
         "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
         "tslib": "^2.8.1",
@@ -67,5 +68,5 @@
             }
         }
     },
-    "gitHead": "9af978adbda8c2a750920e35b50ae19d6e940ddd"
+    "gitHead": "5eab8f877242d46d7cae13b640d8e1adbf194911"
 }