@crawlee/utils 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -2,9 +2,11 @@ export { htmlToText } from './internals/cheerio.js';
 export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
 export { EnqueueStrategy } from './internals/url.js';
 export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
-export { sleep, expandShadowRoots } from './internals/general.js';
+export { sleep } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
-export * from './internals/sitemap.js';
-export * from './internals/validation.js';
+export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
+export type { ParseSitemapOptions } from './internals/sitemap.js';
+export { ArgumentValidationError } from './internals/validation.js';

package/index.js

@@ -1,9 +1,10 @@
 export { htmlToText } from './internals/cheerio.js';
 export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
 export { EnqueueStrategy } from './internals/url.js';
-export { sleep, expandShadowRoots } from './internals/general.js';
+export { sleep } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
-export * from './internals/sitemap.js';
-export * from './internals/validation.js';
+export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
+export { ArgumentValidationError } from './internals/validation.js';

package/internal.d.ts

@@ -1,9 +1,10 @@
 export * from './internals/blocked.js';
-export type { CheerioRoot, CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
+export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
 export { extractUrlsFromCheerio } from './internals/cheerio.js';
 export { tryAbsoluteURL } from './internals/extract-urls.js';
-export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
 export * from './internals/iterables.js';
 export * from './internals/url.js';
 export * from './internals/validation.js';
 export * as schemas from './internals/schemas.js';
+export { parseSitemap, type SitemapUrl } from './internals/sitemap.js';

package/internal.js

@@ -1,8 +1,9 @@
 export * from './internals/blocked.js';
 export { extractUrlsFromCheerio } from './internals/cheerio.js';
 export { tryAbsoluteURL } from './internals/extract-urls.js';
-export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
 export * from './internals/iterables.js';
 export * from './internals/url.js';
 export * from './internals/validation.js';
 export * as schemas from './internals/schemas.js';
+export { parseSitemap } from './internals/sitemap.js';

package/internals/cheerio.d.ts

@@ -1,5 +1,4 @@
 import type { CheerioAPI } from 'cheerio';
-export type CheerioRoot = CheerioAPI;
 export type { CheerioAPI, Cheerio } from 'cheerio';
 export type { Element } from 'domhandler';
 /**
@@ -29,7 +28,7 @@ export type { Element } from 'domhandler';
  * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
  * @return Plain text
  */
-export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;
+export declare function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;
 /**
  * Extracts URLs from a given Cheerio object.
  *

package/internals/cheerio.js

@@ -30,7 +30,7 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
  * @return Plain text
  */
 export async function htmlToText(htmlOrCheerioElement) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     if (!htmlOrCheerioElement)
         return '';
     const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
@@ -42,8 +42,10 @@ export async function htmlToText(htmlOrCheerioElement) {
             if (elem.type === 'text') {
                 // Compress spaces, unless we're inside <pre> element
                 let compr;
-                if (elem.parent?.tagName === 'pre')
-                    compr = elem.data;
+                if (elem.parent?.tagName === 'pre') {
+                    // Strip single newline right after `<pre>` that htmlparser2 keeps
+                    compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
+                }
                 else
                     compr = elem.data.replace(/\s+/g, ' ');
                 // If text is empty or ends with a whitespace, don't add the leading whitespace

package/internals/extract-microdata.d.ts

@@ -0,0 +1,24 @@
+import type { CheerioAPI } from 'cheerio';
+/** The value of a microdata property: either text or a nested item. */
+export type MicrodataValue = string | MicrodataItem;
+/** A single schema.org item extracted from a document. */
+export interface MicrodataItem {
+    /** Tokens of the item's `itemtype` attribute. */
+    type?: string[];
+    /** The item's `itemid` attribute. */
+    id?: string;
+    /** Values keyed by `itemprop` name, an array where the property repeats. */
+    properties: Record<string, MicrodataValue | MicrodataValue[]>;
+}
+/**
+ * Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,
+ * following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).
+ *
+ * Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned
+ * verbatim rather than resolved against the document's base URL.
+ *
+ * @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.
+ * @returns The document's top-level items. Nested items are the property values of their parent.
+ */
+export declare function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
+export declare function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;

package/internals/extract-microdata.js

@@ -0,0 +1,118 @@
+import { isTag } from 'domhandler';
+export async function extractMicrodata(htmlOrCheerioElement) {
+    // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
+    const { load } = await import('cheerio/slim');
+    const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
+    const context = { $ };
+    return $('[itemscope]')
+        .toArray()
+        .filter((element) => !('itemprop' in element.attribs))
+        .map((element) => parseItem(context, element, new Set()));
+}
+function parseItem(context, element, ancestors) {
+    const item = { properties: {} };
+    const type = uniqueTokens(element.attribs.itemtype);
+    const id = element.attribs.itemid;
+    if (type.length > 0) {
+        item.type = type;
+    }
+    if (id) {
+        item.id = id.trim();
+    }
+    ancestors.add(element);
+    for (const propertyElement of collectPropertyElements(context, element)) {
+        const value = getPropertyValue(context, propertyElement, ancestors);
+        for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {
+            addProperty(item.properties, name, value);
+        }
+    }
+    ancestors.delete(element);
+    return item;
+}
+function collectPropertyElements(context, scope) {
+    const elements = [];
+    collectFromNodes(scope.children, elements);
+    for (const id of uniqueTokens(scope.attribs.itemref)) {
+        const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
+        if (referenced) {
+            collectFromNodes([referenced], elements);
+        }
+    }
+    return elements;
+}
+function collectFromNodes(nodes, elements) {
+    for (const node of nodes) {
+        if (!isTag(node)) {
+            continue;
+        }
+        if ('itemprop' in node.attribs) {
+            elements.push(node);
+        }
+        // A nested item owns everything below it, so its subtree is not part of the enclosing item.
+        if (!('itemscope' in node.attribs)) {
+            collectFromNodes(node.children, elements);
+        }
+    }
+}
+function indexIds($) {
+    const index = new Map();
+    for (const element of $('[id]').toArray()) {
+        // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
+        if (!index.has(element.attribs.id)) {
+            index.set(element.attribs.id, element);
+        }
+    }
+    return index;
+}
+function getPropertyValue(context, element, ancestors) {
+    if ('itemscope' in element.attribs) {
+        // `itemref` can point back at an enclosing item, which the spec treats as an error.
+        return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);
+    }
+    const { attribs } = element;
+    switch (element.tagName.toLowerCase()) {
+        case 'meta':
+            return attribs.content ?? '';
+        case 'audio':
+        case 'embed':
+        case 'iframe':
+        case 'img':
+        case 'source':
+        case 'track':
+        case 'video':
+            return attribs.src ?? '';
+        case 'a':
+        case 'area':
+        case 'link':
+            return attribs.href ?? '';
+        case 'object':
+            return attribs.data ?? '';
+        case 'data':
+        case 'meter':
+            return attribs.value ?? '';
+        case 'time':
+            return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();
+        default:
+            return context.$(element).text().replace(/\s+/g, ' ').trim();
+    }
+}
+function addProperty(properties, name, value) {
+    const existing = properties[name];
+    if (existing === undefined) {
+        properties[name] = value;
+    }
+    else if (Array.isArray(existing)) {
+        existing.push(value);
+    }
+    else {
+        properties[name] = [existing, value];
+    }
+}
+/** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */
+function uniqueTokens(value) {
+    if (!value) {
+        return [];
+    }
+    const tokens = value.split(/\s+/).filter(Boolean);
+    return tokens.length > 1 ? [...new Set(tokens)] : tokens;
+}

package/internals/open_graph_parser.js

@@ -366,7 +366,7 @@ const parseOpenGraphProperty = (property, $) => {
     };
 };
 export async function parseOpenGraph(item, additionalProperties) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof item === 'string' ? load(item) : item;
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
         return {

package/internals/robots.d.ts

@@ -53,7 +53,6 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    private static load;
     /**
      * Get crawl delay for a given user agent.
      * @param [userAgent] relevant user agent, default to `*`

package/internals/robots.js

@@ -43,7 +43,7 @@ export class RobotsTxtFile {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
+        return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -55,20 +55,19 @@ export class RobotsTxtFile {
         // @ts-ignore
         return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
-    static async load(url, options) {
+    static async #load(url, options) {
         const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
         const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
             proxyUrl,
             timeoutMillis: options?.timeoutMillis,
             signal: options?.signal,
         });
-        if (response.status < 200 || response.status >= 300) {
-            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
-        }
-        if (response.status === 404) {
+        // RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
+        if (response.status >= 400 && response.status < 600) {
+            const allowed = response.status < 500;
             return new RobotsTxtFile(url, {
                 isAllowed() {
-                    return true;
+                    return allowed;
                 },
                 getSitemaps() {
                     return [];
@@ -78,6 +77,9 @@ export class RobotsTxtFile {
                 },
             }, proxyUrl, logger);
         }
+        if (response.status < 200 || response.status >= 300) {
+            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
+        }
         // @ts-ignore
         return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }

package/internals/schemas.d.ts

@@ -30,7 +30,7 @@ export declare function objectWithKeys(keys: string[], message?: string): z.ZodT
  * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
  * @internal
  */
-export declare const httpClient: z.ZodCustom<BaseHttpClient, BaseHttpClient>;
+export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
 /**
  * Accepts any object implementing the CrawleeLogger interface.
  * @internal
@@ -47,7 +47,7 @@ export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>,
  */
 export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
 /**
- * Shape of a request stored in a request queue.
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
  * @internal
  */
 export declare const storageRequest: z.ZodObject<{

package/internals/schemas.js

@@ -54,11 +54,7 @@ export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(val
  * @internal
  */
 export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
-/**
- * Shape of a request stored in a request queue.
- * @internal
- */
-export const storageRequest = z.looseObject({
+const storageRequestShape = z.looseObject({
     id: z.string(),
     url: z.url({ protocol: /^https?$/ }),
     uniqueKey: z.string(),
@@ -66,11 +62,16 @@ export const storageRequest = z.looseObject({
     retryCount: z.number().int().optional(),
     handledAt: z.union([z.string(), z.date()]).optional(),
 });
+/**
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
+ * @internal
+ */
+export const storageRequest = z.compile(storageRequestShape);
 /**
  * {@link storageRequest} before an id is assigned.
  * @internal
  */
-export const storageRequestWithoutId = storageRequest.omit({ id: true });
+export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
 /**
  * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
  * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
@@ -86,7 +87,7 @@ export function arrayOf(item, elements) {
  * Batch of {@link storageRequestWithoutId}.
  * @internal
  */
-export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');
+export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
 /**
  * Options of request queue add/update operations.
  * @internal

package/internals/sitemap.d.ts

@@ -82,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
  * ```
  */
 export declare class Sitemap {
+    #private;
     readonly urls: string[];
     constructor(urls: string[]);
     /**
@@ -103,7 +104,6 @@ export declare class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
-    private static parse;
 }
 /**
  * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,

package/internals/sitemap.js

@@ -3,7 +3,7 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
 import { StringDecoder } from 'node:string_decoder';
 import { createGunzip } from 'node:zlib';
 import { FetchHttpClient } from '@crawlee/http-client';
-import MIMEType from 'whatwg-mimetype';
+import { MIMEType } from 'whatwg-mimetype';
 import { mergeAsyncIterables } from './iterables.js';
 import { RobotsTxtFile } from './robots.js';
 import { filterUrl } from './url.js';
@@ -14,16 +14,16 @@ class SitemapTxtParser extends Transform {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.processBuffer(this.#decoder.write(chunk), false);
+                this.#processBuffer(this.#decoder.write(chunk), false);
                 callback();
             },
             flush: (callback) => {
-                this.processBuffer(this.#decoder.end(), true);
+                this.#processBuffer(this.#decoder.end(), true);
                 callback();
             },
         });
     }
-    processBuffer(input, finalize) {
+    #processBuffer(input, finalize) {
         this.#buffer += input;
         if (finalize || this.#buffer.includes('\n')) {
             const parts = this.#buffer
@@ -52,7 +52,7 @@ class SitemapXmlParser extends Transform {
     #currentTag = undefined;
     #url = {};
     static async create() {
-        const { SAXParser } = await import('sax');
+        const { SAXParser } = (await import('sax')).default;
         return new SitemapXmlParser(new SAXParser(true));
     }
     constructor(parser) {
@@ -72,13 +72,13 @@ class SitemapXmlParser extends Transform {
             },
         });
         this.#parser = parser;
-        this.#parser.onopentag = this.onOpenTag.bind(this);
-        this.#parser.onclosetag = this.onCloseTag.bind(this);
-        this.#parser.ontext = this.onText.bind(this);
-        this.#parser.oncdata = this.onText.bind(this);
+        this.#parser.onopentag = this.#onOpenTag.bind(this);
+        this.#parser.onclosetag = this.#onCloseTag.bind(this);
+        this.#parser.ontext = this.#onText.bind(this);
+        this.#parser.oncdata = this.#onText.bind(this);
         this.#parser.onerror = this.destroy.bind(this);
     }
-    onOpenTag(node) {
+    #onOpenTag(node) {
         if (this.#rootTagName !== undefined) {
             if (node.name === 'loc' ||
                 node.name === 'lastmod' ||
@@ -94,7 +94,7 @@ class SitemapXmlParser extends Transform {
             this.#rootTagName = 'sitemapindex';
         }
     }
-    onCloseTag(name) {
+    #onCloseTag(name) {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
             this.#currentTag = undefined;
         }
@@ -105,7 +105,7 @@ class SitemapXmlParser extends Transform {
             this.#url = {};
         }
     }
-    onText(text) {
+    #onText(text) {
         if (this.#currentTag === 'loc') {
             if (this.#rootTagName === 'sitemapindex') {
                 this.push({ type: 'sitemapUrl', url: text.trim() });
@@ -323,7 +323,7 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async load(urls, proxyUrl, parseSitemapOptions) {
-        return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
+        return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
     }
     /**
      * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -331,9 +331,9 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
-        return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
+        return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
     }
-    static async parse(sources, proxyUrl, parseSitemapOptions) {
+    static async #parse(sources, proxyUrl, parseSitemapOptions) {
         const urls = [];
         try {
             for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
@@ -423,6 +423,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
         }
         else {
             const firstUrl = new URL(domainUrls[0]);
+            // The guessed candidates are well-known paths on the same origin, so the query string
+            // and fragment of the input URL are not part of them - carrying them over probes (and
+            // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
+            firstUrl.search = '';
+            firstUrl.hash = '';
             const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
             const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
                 firstUrl.pathname = pathname;

package/internals/social.d.ts

@@ -225,7 +225,7 @@ export declare const TWITTER_REGEX: RegExp;
  * ```
  * import { social } from 'crawlee';
  *
- * const matches = text.match(social.TWITTER_REGEX_STRING);
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
  * ```
  */

package/internals/social.js

@@ -326,7 +326,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
  * ```
  * import { social } from 'crawlee';
  *
- * const matches = text.match(social.TWITTER_REGEX_STRING);
+ * const matches = text.match(social.TWITTER_REGEX_GLOBAL);
  * if (matches) console.log(`${matches.length} Twitter profiles found!`);
  * ```
  */
@@ -590,7 +590,7 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
  * @return An object with the social handles.
  */
 export async function parseHandlesFromHtml(html, data = null) {
-    const cheerio = await import('cheerio');
+    const cheerio = await import('cheerio/slim');
     const result = {
         emails: [],
         phones: [],

package/internals/validation.d.ts

@@ -1,25 +1,9 @@
-import type { z } from 'zod';
+import type { Dictionary } from '@crawlee/types';
+export { ArgumentValidationError, parseArgument } from '@apify/validations';
 /**
- * Thrown when an argument fails schema validation.
- *
- * Its `message` is a human-readable sentence naming the offending field and the
- * value it received (rather than a raw JSON dump). The structured
- * {@link https://zod.dev | zod} issues are available on `issues`, and the
- * original `ZodError` on `cause`, for programmatic inspection.
- */
-export declare class ArgumentValidationError extends Error {
-    /** Structured issues from the underlying schema check. */
-    readonly issues: z.ZodError['issues'];
-    /** The raw zod error that triggered this. */
-    readonly cause: z.ZodError;
-    constructor(error: z.ZodError, value: unknown, label?: string);
-}
-/**
- * Parses `value` with `schema`, returning the typed result (with schema defaults applied).
- * Throws {@link ArgumentValidationError} on failure.
- *
- * The optional `label` names the interface being validated and is appended to every error line
- * (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
+ * into a second, half-working way of configuring the same pool.
  * @internal
  */
-export declare function parseArgument<TValue, TSchema extends z.ZodType>(value: TValue, schema: TSchema, label?: string): TValue & z.output<TSchema>;
+export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;

package/internals/validation.js

@@ -1,140 +1,16 @@
-/** Formats a zod issue path like `groups[0]` or `countryCode`. */
-function formatIssuePath(path) {
-    let out = '';
-    for (const key of path) {
-        if (typeof key === 'number')
-            out += `[${key}]`;
-        else
-            out += out ? `.${String(key)}` : String(key);
-    }
-    return out;
-}
-/** Reads the value at `path` from the validated input, to include in the error. */
-function valueAtPath(root, path) {
-    let current = root;
-    for (const key of path) {
-        if (current === null || typeof current !== 'object')
-            return undefined;
-        current = current[key];
-    }
-    return current;
-}
-/** Names the runtime type of `value` the way zod's own messages do (`null`, `array`, `string`, …). */
-function describeType(value) {
-    if (value === null)
-        return 'null';
-    if (Array.isArray(value))
-        return 'array';
-    return typeof value;
-}
-/** The bare custom-schema messages that stop at the expected type, e.g. `Invalid input: expected number`. */
-const BARE_EXPECTED_TYPE_MESSAGE = /^Invalid input: expected (an array of .+|a typed array|an object|object|array|function|number|string|boolean)$/;
-/** Longest received string rendered in an error; the rest is elided. */
-const MAX_RENDERED_STRING_LENGTH = 200;
-/** Renders a primitive received value for an error; skips objects/Dates (noisy). */
-function describeReceived(value) {
-    switch (typeof value) {
-        case 'string':
-            // An empty string would render as bare backticks — make it visible.
-            if (value === '')
-                return "''";
-            return value.length > MAX_RENDERED_STRING_LENGTH
-                ? `${value.slice(0, MAX_RENDERED_STRING_LENGTH)}… (${value.length - MAX_RENDERED_STRING_LENGTH} more characters)`
-                : value;
-        case 'number':
-        case 'boolean':
-        case 'bigint':
-            return String(value);
-        default:
-            return undefined;
-    }
-}
-/** Renders the received side of a sentence: ``received the string `abc` ``, `received NaN`, `received array`. */
-function describeReceivedClause(value) {
-    if (typeof value === 'number' && Number.isNaN(value))
-        return 'received NaN';
-    if (value === '')
-        return 'received an empty string';
-    const rendered = describeReceived(value);
-    return rendered === undefined
-        ? `received ${describeType(value)}`
-        : `received the ${describeType(value)} \`${rendered}\``;
-}
-/** Renders one issue as a line each; a union expands into a line per failed arm. */
-function formatIssue(issue, root, basePath) {
-    const path = [...basePath, ...issue.path];
-    // A union's own message is a bare "Invalid input" — the useful part is in `errors`,
-    // whose paths are relative to the union, hence passing `path` down as the base.
-    if (issue.code === 'invalid_union') {
-        return issue.errors.flatMap((arm) => arm.flatMap((nested) => formatIssue(nested, root, path)));
-    }
-    const location = path.length ? ` at \`${formatIssuePath(path)}\`` : '';
-    const value = valueAtPath(root, path);
-    const rendered = describeReceived(value);
-    // ow named the received type ("expected `number` but received type `string`"). The received value is
-    // folded into that clause (``received the string `3` ``) rather than dangling after the location: our
-    // custom schemas stop at the expected type, so the clause is appended; zod's built-in messages already
-    // end with `, received <type>`, so that tail is replaced with the enriched one.
-    let { message } = issue;
-    let got = '';
-    const bareExpected = BARE_EXPECTED_TYPE_MESSAGE.exec(message);
-    const zodReceived = /, received (\S+)$/.exec(message);
-    // `arrayOf` messages name the element type — their expected runtime type is `array`.
-    const expectedType = bareExpected?.[1].startsWith('an array of') ? 'array' : bareExpected?.[1];
-    if (bareExpected && expectedType !== (Number.isNaN(value) ? 'NaN' : describeType(value))) {
-        message += `, ${describeReceivedClause(value)}`;
-    }
-    else if (zodReceived && zodReceived[1] === describeType(value) && rendered !== undefined) {
-        message = `${message.slice(0, zodReceived.index)}, ${describeReceivedClause(value)}`;
-    }
-    else if (rendered !== undefined && !message.endsWith(`received ${rendered}`)) {
-        // Messages that never name a received type (regex, min/max, enums) keep the plain value suffix.
-        got = `, got \`${rendered}\``;
-    }
-    return [`${message}${location}${got}`];
-}
-/**
- * Formats a `ZodError` as a plain, human-readable message that names the
- * offending field *and* the value it received (e.g. ``must match pattern
- * /^[A-Z]{2}$/ at `countryCode`, got `CZE` ``) — closer to the old `ow` errors
- * than zod's default, which omits the received value.
- */
-function formatZodError(error, root, label) {
-    const lines = error.issues.flatMap((issue) => formatIssue(issue, root, []));
-    // The label names the validated interface, the way ow's errors ended with "in object `X`".
-    return (label ? lines.map((line) => `${line} in \`${label}\``) : lines).join('\n');
-}
-/**
- * Thrown when an argument fails schema validation.
- *
- * Its `message` is a human-readable sentence naming the offending field and the
- * value it received (rather than a raw JSON dump). The structured
- * {@link https://zod.dev | zod} issues are available on `issues`, and the
- * original `ZodError` on `cause`, for programmatic inspection.
- */
-export class ArgumentValidationError extends Error {
-    /** Structured issues from the underlying schema check. */
-    issues;
-    /** The raw zod error that triggered this. */
-    cause;
-    constructor(error, value, label) {
-        super(formatZodError(error, value, label), { cause: error });
-        this.name = 'ArgumentValidationError';
-        this.issues = error.issues;
-        this.cause = error;
-    }
-}
+export { ArgumentValidationError, parseArgument } from '@apify/validations';
 /**
- * Parses `value` with `schema`, returning the typed result (with schema defaults applied).
- * Throws {@link ArgumentValidationError} on failure.
- *
- * The optional `label` names the interface being validated and is appended to every error line
- * (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
+ * into a second, half-working way of configuring the same pool.
  * @internal
  */
-export function parseArgument(value, schema, label) {
-    const result = schema.safeParse(value);
-    if (!result.success)
-        throw new ArgumentValidationError(result.error, value, label);
-    return result.data;
+export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
+    const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
+    if (names.length === 0) {
+        return;
+    }
+    throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
+        `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
+        'itself. Configure the pool you pass in instead.');
 }

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -42,19 +42,19 @@
         "copy": "tsx ../../scripts/copy.ts"
     },
     "dependencies": {
-        "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
+        "@apify/validations": "^1.1.0",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
-        "domhandler": "^5.0.3",
+        "domhandler": "^6.0.0",
         "file-type": "^21.0.0",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
-        "whatwg-mimetype": "^4.0.0",
-        "zod": "^4.4.3"
+        "whatwg-mimetype": "^5.0.0",
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }