@crawlee/utils 4.0.0-beta.98 → 4.0.0-rc.0

package/index.d.ts

@@ -1,10 +1,10 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
+export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
+export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';
+export * from './internals/validation.js';

package/index.js

@@ -1,10 +1,9 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
+export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';
+export * from './internals/validation.js';

package/internal.d.ts

@@ -0,0 +1,9 @@
+export * from './internals/blocked.js';
+export type { CheerioRoot, CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';
+export * from './internals/validation.js';
+export * as schemas from './internals/schemas.js';

package/internal.js

@@ -0,0 +1,8 @@
+export * from './internals/blocked.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';
+export * from './internals/validation.js';
+export * as schemas from './internals/schemas.js';

package/internals/extract-urls.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
 export interface DownloadListOfUrlsOptions {
     /**
      * URL to the file

package/internals/extract-urls.js

@@ -1,19 +1,25 @@
 import { FetchHttpClient } from '@crawlee/http-client';
-import ow from 'ow';
+import { z } from 'zod';
 import { URL_NO_COMMAS_REGEX } from './general.js';
+import { httpClient as httpClientSchema } from './schemas.js';
+import { parseArgument } from './validation.js';
+const downloadListOfUrlsOptionsSchema = z.strictObject({
+    url: z.url(),
+    encoding: z.string().default('utf8'),
+    urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
+    proxyUrl: z.string().optional(),
+    httpClient: httpClientSchema.default(() => new FetchHttpClient()),
+});
+const extractUrlsOptionsSchema = z.strictObject({
+    string: z.string(),
+    urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
+});
 /**
  * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
  * Optionally, custom regular expression and encoding may be provided.
  */
 export async function downloadListOfUrls(options) {
-    ow(options, ow.object.exactShape({
-        url: ow.string.url,
-        encoding: ow.optional.string,
-        urlRegExp: ow.optional.regExp,
-        proxyUrl: ow.optional.string,
-        httpClient: ow.optional.object,
-    }));
-    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new FetchHttpClient(), } = options;
+    const { url, encoding, urlRegExp, proxyUrl, httpClient } = parseArgument(options, downloadListOfUrlsOptionsSchema);
     // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
     const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
     let fixedUrl = url;
@@ -30,13 +36,9 @@ export async function downloadListOfUrls(options) {
  * Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
  */
 export function extractUrls(options) {
-    ow(options, ow.object.exactShape({
-        string: ow.string,
-        urlRegExp: ow.optional.regExp,
-    }));
-    const lines = options.string.split('\n');
+    const { string, urlRegExp } = parseArgument(options, extractUrlsOptionsSchema);
+    const lines = string.split('\n');
     const result = [];
-    const urlRegExp = options.urlRegExp ?? URL_NO_COMMAS_REGEX;
     for (const line of lines) {
         result.push(...(line.match(urlRegExp) ?? []));
     }

package/internals/robots.d.ts

@@ -1,5 +1,15 @@
-import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
+import { type EnqueueStrategy } from './url.js';
+export interface RobotsTxtFileSitemapsOptions {
+    /**
+     * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+     * are always dropped. Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -19,9 +29,7 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export declare class RobotsTxtFile {
-    private robots;
-    private proxyUrl?;
-    private logger?;
+    #private;
     private constructor();
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
@@ -46,6 +54,11 @@ export declare class RobotsTxtFile {
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
     private static load;
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent?: string): number | undefined;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt
@@ -53,15 +66,19 @@ export declare class RobotsTxtFile {
      */
     isAllowed(url: string, userAgent?: string): boolean;
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps(): string[];
+    getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    parseSitemaps(): Promise<Sitemap>;
+    parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    parseUrlsFromSitemaps(): Promise<string[]>;
+    parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
 }

package/internals/robots.js

@@ -1,6 +1,7 @@
 import { FetchHttpClient } from '@crawlee/http-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
+import { filterUrl } from './url.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -20,13 +21,15 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export class RobotsTxtFile {
-    robots;
-    proxyUrl;
-    logger;
-    constructor(robots, proxyUrl, logger) {
-        this.robots = robots;
-        this.proxyUrl = proxyUrl;
-        this.logger = logger;
+    #url;
+    #robots;
+    #proxyUrl;
+    #logger;
+    constructor(url, robots, proxyUrl, logger) {
+        this.#url = url;
+        this.#robots = robots;
+        this.#proxyUrl = proxyUrl;
+        this.#logger = logger;
     }
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
@@ -50,7 +53,7 @@ export class RobotsTxtFile {
      */
     static from(url, content, proxyUrl) {
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
+        return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
     static async load(url, options) {
         const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
@@ -63,17 +66,27 @@ export class RobotsTxtFile {
             throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
         }
         if (response.status === 404) {
-            return new RobotsTxtFile({
+            return new RobotsTxtFile(url, {
                 isAllowed() {
                     return true;
                 },
                 getSitemaps() {
                     return [];
                 },
+                getCrawlDelay() {
+                    return undefined;
+                },
             }, proxyUrl, logger);
         }
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
+        return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
+    }
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent = '*') {
+        return this.#robots.getCrawlDelay(userAgent);
     }
     /**
      * Check if a URL should be crawled by robots.
@@ -81,24 +94,39 @@ export class RobotsTxtFile {
      * @param [userAgent] relevant user agent, default to `*`
      */
     isAllowed(url, userAgent = '*') {
-        return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
+        return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
     }
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps() {
-        return this.robots.getSitemaps();
+    getSitemaps(options = {}) {
+        const { enqueueStrategy = 'same-hostname' } = options;
+        const sitemaps = [];
+        for (const sitemapUrl of this.#robots.getSitemaps()) {
+            // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+            const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
+            if (!allowed) {
+                this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
+                continue;
+            }
+            sitemaps.push(sitemapUrl);
+        }
+        return sitemaps;
     }
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    async parseSitemaps() {
-        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl, { logger: this.logger });
+    async parseSitemaps(options = {}) {
+        return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    async parseUrlsFromSitemaps() {
-        return (await this.parseSitemaps()).urls;
+    async parseUrlsFromSitemaps(options = {}) {
+        return (await this.parseSitemaps(options)).urls;
     }
 }

package/internals/schemas.d.ts

@@ -0,0 +1,114 @@
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
+import { z } from 'zod';
+/**
+ * Accepts any object (including arrays and functions).
+ * @internal
+ */
+export declare const anyObject: z.ZodCustom<Dictionary, Dictionary>;
+/**
+ * Accepts any array without validating its items (cheap for huge arrays).
+ * @internal
+ */
+export declare const anyArray: z.ZodCustom<unknown[], unknown[]>;
+/**
+ * Accepts any function.
+ * @internal
+ */
+export declare const anyFunction: z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>;
+/**
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
+ * @internal
+ */
+export declare const anyNumber: z.ZodCustom<number, number>;
+/**
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
+ * @internal
+ */
+export declare function objectWithKeys(keys: string[], message?: string): z.ZodType<Dictionary>;
+/**
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
+ * @internal
+ */
+export declare const httpClient: z.ZodCustom<BaseHttpClient, BaseHttpClient>;
+/**
+ * Accepts any object implementing the CrawleeLogger interface.
+ * @internal
+ */
+export declare const logger: z.ZodType<Dictionary, unknown, z.core.$ZodTypeInternals<Dictionary, unknown>>;
+/**
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
+ * @internal
+ */
+export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>, NodeJS.TypedArray<ArrayBufferLike>>;
+/**
+ * Accepts any non-null, non-array object.
+ * @internal
+ */
+export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
+/**
+ * Shape of a request stored in a request queue.
+ * @internal
+ */
+export declare const storageRequest: z.ZodObject<{
+    id: z.ZodString;
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>;
+/**
+ * {@link storageRequest} before an id is assigned.
+ * @internal
+ */
+export declare const storageRequestWithoutId: z.ZodObject<{
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>;
+/**
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
+ * @internal
+ */
+export declare function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string): z.ZodArray<TItem>;
+/**
+ * Batch of {@link storageRequestWithoutId}.
+ * @internal
+ */
+export declare const storageRequestBatch: z.ZodArray<z.ZodObject<{
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>>;
+/**
+ * Options of request queue add/update operations.
+ * @internal
+ */
+export declare const requestQueueOperationOptions: z.ZodObject<{
+    forefront: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+/**
+ * Options of key-value store `listKeys`.
+ * @internal
+ */
+export declare const keyValueStoreListKeysOptions: z.ZodObject<{
+    prefix: z.ZodOptional<z.ZodString>;
+    exclusiveStartKey: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * Options of dataset item listing.
+ * @internal
+ */
+export declare const datasetListItemsOptions: z.ZodObject<{
+    desc: z.ZodOptional<z.ZodBoolean>;
+    limit: z.ZodOptional<z.ZodNumber>;
+    offset: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;

package/internals/schemas.js

@@ -0,0 +1,114 @@
+import { BaseHttpClient } from '@crawlee/http-client';
+import { z } from 'zod';
+/**
+ * Accepts any object (including arrays and functions).
+ * @internal
+ */
+export const anyObject = z.custom((value) => (typeof value === 'object' && value !== null) || typeof value === 'function', { message: 'Invalid input: expected object' });
+/**
+ * Accepts any array without validating its items (cheap for huge arrays).
+ * @internal
+ */
+export const anyArray = z.custom(Array.isArray, { message: 'Invalid input: expected array' });
+/**
+ * Accepts any function.
+ * @internal
+ */
+export const anyFunction = z.custom((value) => typeof value === 'function', {
+    message: 'Invalid input: expected function',
+});
+/**
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
+ * @internal
+ */
+export const anyNumber = z.custom((value) => typeof value === 'number' && !Number.isNaN(value), {
+    message: 'Invalid input: expected number',
+});
+/**
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
+ * @internal
+ */
+export function objectWithKeys(keys, message) {
+    return z.custom((value) => ((typeof value === 'object' && value !== null) || typeof value === 'function') &&
+        keys.every((key) => key in value), {
+        message: message ?? `Invalid input: expected an object with keys ${keys.map((key) => `'${key}'`).join(', ')}`,
+    });
+}
+/**
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
+ * @internal
+ */
+export const httpClient = z.instanceof(BaseHttpClient);
+/**
+ * Accepts any object implementing the CrawleeLogger interface.
+ * @internal
+ */
+export const logger = objectWithKeys(['child', 'info', 'error', 'warning'], "Expected an object implementing the CrawleeLogger interface (missing one of 'child', 'info', 'error', 'warning'), got something else.");
+/**
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
+ * @internal
+ */
+export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(value instanceof DataView), { message: 'Invalid input: expected a typed array' });
+/**
+ * Accepts any non-null, non-array object.
+ * @internal
+ */
+export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
+/**
+ * Shape of a request stored in a request queue.
+ * @internal
+ */
+export const storageRequest = z.looseObject({
+    id: z.string(),
+    url: z.url({ protocol: /^https?$/ }),
+    uniqueKey: z.string(),
+    method: z.string().optional(),
+    retryCount: z.number().int().optional(),
+    handledAt: z.union([z.string(), z.date()]).optional(),
+});
+/**
+ * {@link storageRequest} before an id is assigned.
+ * @internal
+ */
+export const storageRequestWithoutId = storageRequest.omit({ id: true });
+/**
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
+ * @internal
+ */
+export function arrayOf(item, elements) {
+    return z.array(item, {
+        error: (issue) => issue.code === 'invalid_type' ? `Invalid input: expected an array of ${elements}` : undefined,
+    });
+}
+/**
+ * Batch of {@link storageRequestWithoutId}.
+ * @internal
+ */
+export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');
+/**
+ * Options of request queue add/update operations.
+ * @internal
+ */
+export const requestQueueOperationOptions = z.object({
+    forefront: z.boolean().optional(),
+});
+/**
+ * Options of key-value store `listKeys`.
+ * @internal
+ */
+export const keyValueStoreListKeysOptions = z.object({
+    prefix: z.string().optional(),
+    exclusiveStartKey: z.string().optional(),
+    limit: z.number().int().gt(0).optional(),
+});
+/**
+ * Options of dataset item listing.
+ * @internal
+ */
+export const datasetListItemsOptions = z.object({
+    desc: z.boolean().optional(),
+    limit: z.number().int().optional(),
+    offset: z.number().int().optional(),
+});

package/internals/sitemap.d.ts

@@ -1,4 +1,6 @@
-import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
+import { type EnqueueStrategy } from './url.js';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -54,6 +56,13 @@ export interface ParseSitemapOptions {
      * If not provided, all nested sitemaps are followed.
      */
     nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+     * sources (no parent URL). Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Optional logger for reporting warnings during sitemap parsing.
      */

package/internals/sitemap.js

@@ -3,31 +3,30 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
 import { StringDecoder } from 'node:string_decoder';
 import { createGunzip } from 'node:zlib';
 import { FetchHttpClient } from '@crawlee/http-client';
-import { fileTypeStream } from 'file-type';
-import sax from 'sax';
 import MIMEType from 'whatwg-mimetype';
 import { mergeAsyncIterables } from './iterables.js';
 import { RobotsTxtFile } from './robots.js';
+import { filterUrl } from './url.js';
 class SitemapTxtParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    buffer = '';
+    #decoder = new StringDecoder('utf8');
+    #buffer = '';
     constructor() {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.processBuffer(this.decoder.write(chunk), false);
+                this.processBuffer(this.#decoder.write(chunk), false);
                 callback();
             },
             flush: (callback) => {
-                this.processBuffer(this.decoder.end(), true);
+                this.processBuffer(this.#decoder.end(), true);
                 callback();
             },
         });
     }
     processBuffer(input, finalize) {
-        this.buffer += input;
-        if (finalize || this.buffer.includes('\n')) {
-            const parts = this.buffer
+        this.#buffer += input;
+        if (finalize || this.#buffer.includes('\n')) {
+            const parts = this.#buffer
                 .split('\n')
                 .map((part) => part.trim())
                 .filter((part) => part.length > 0);
@@ -35,104 +34,109 @@ class SitemapTxtParser extends Transform {
                 for (const url of parts) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = '';
+                this.#buffer = '';
             }
             else if (parts.length > 0) {
                 for (const url of parts.slice(0, -1)) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = parts.at(-1);
+                this.#buffer = parts.at(-1);
             }
         }
     }
 }
 class SitemapXmlParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    parser = new sax.SAXParser(true);
-    rootTagName;
-    currentTag = undefined;
-    url = {};
-    constructor() {
+    #decoder = new StringDecoder('utf8');
+    #parser;
+    #rootTagName;
+    #currentTag = undefined;
+    #url = {};
+    static async create() {
+        const { SAXParser } = await import('sax');
+        return new SitemapXmlParser(new SAXParser(true));
+    }
+    constructor(parser) {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.parser.write(this.decoder.write(chunk));
+                this.#parser.write(this.#decoder.write(chunk));
                 callback();
             },
             flush: (callback) => {
-                const rest = this.decoder.end();
+                const rest = this.#decoder.end();
                 if (rest.length > 0) {
-                    this.parser.write(rest);
+                    this.#parser.write(rest);
                 }
-                this.parser.end();
+                this.#parser.end();
                 callback();
             },
         });
-        this.parser.onopentag = this.onOpenTag.bind(this);
-        this.parser.onclosetag = this.onCloseTag.bind(this);
-        this.parser.ontext = this.onText.bind(this);
-        this.parser.oncdata = this.onText.bind(this);
-        this.parser.onerror = this.destroy.bind(this);
+        this.#parser = parser;
+        this.#parser.onopentag = this.onOpenTag.bind(this);
+        this.#parser.onclosetag = this.onCloseTag.bind(this);
+        this.#parser.ontext = this.onText.bind(this);
+        this.#parser.oncdata = this.onText.bind(this);
+        this.#parser.onerror = this.destroy.bind(this);
     }
     onOpenTag(node) {
-        if (this.rootTagName !== undefined) {
+        if (this.#rootTagName !== undefined) {
             if (node.name === 'loc' ||
                 node.name === 'lastmod' ||
                 node.name === 'priority' ||
                 node.name === 'changefreq') {
-                this.currentTag = node.name;
+                this.#currentTag = node.name;
             }
         }
         if (node.name === 'urlset') {
-            this.rootTagName = 'urlset';
+            this.#rootTagName = 'urlset';
         }
         if (node.name === 'sitemapindex') {
-            this.rootTagName = 'sitemapindex';
+            this.#rootTagName = 'sitemapindex';
         }
     }
     onCloseTag(name) {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
-            this.currentTag = undefined;
+            this.#currentTag = undefined;
         }
         if (name === 'url') {
-            if (this.url.loc !== undefined) {
-                this.push({ type: 'url', ...this.url, loc: this.url.loc });
+            if (this.#url.loc !== undefined) {
+                this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
             }
-            this.url = {};
+            this.#url = {};
         }
     }
     onText(text) {
-        if (this.currentTag === 'loc') {
-            if (this.rootTagName === 'sitemapindex') {
+        if (this.#currentTag === 'loc') {
+            if (this.#rootTagName === 'sitemapindex') {
                 this.push({ type: 'sitemapUrl', url: text.trim() });
             }
-            if (this.rootTagName === 'urlset') {
-                this.url ??= {};
-                this.url.loc = text.trim();
+            if (this.#rootTagName === 'urlset') {
+                this.#url ??= {};
+                this.#url.loc = text.trim();
             }
         }
         text = text.trim();
-        if (this.currentTag === 'lastmod') {
+        if (this.#currentTag === 'lastmod') {
             const lastmod = new Date(text);
             if (!Number.isNaN(lastmod.getTime())) {
-                this.url.lastmod = lastmod;
+                this.#url.lastmod = lastmod;
             }
         }
-        if (this.currentTag === 'priority') {
-            this.url.priority = Number(text);
+        if (this.#currentTag === 'priority') {
+            this.#url.priority = Number(text);
         }
-        if (this.currentTag === 'changefreq') {
+        if (this.#currentTag === 'changefreq') {
             if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
-                this.url.changefreq = text;
+                this.#url.changefreq = text;
             }
         }
     }
 }
 export async function* parseSitemap(initialSources, proxyUrl, options) {
-    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
+    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
-    const createParser = (contentType = '', url) => {
+    const createParser = async (contentType = '', url) => {
         let mimeType;
         try {
             mimeType = new MIMEType(contentType);
@@ -141,7 +145,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             mimeType = null;
         }
         if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
-            return new SitemapXmlParser();
+            return SitemapXmlParser.create();
         }
         if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
             return new SitemapTxtParser();
@@ -154,8 +158,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             continue;
         }
         let items = null;
+        // Parent URL, parsed once and reused as the origin for the strategy checks below.
+        let sitemapUrl;
         if (source.type === 'url') {
-            const sitemapUrl = new URL(source.url);
+            sitemapUrl = new URL(source.url);
             visitedSitemapUrls.add(sitemapUrl.toString());
             let retriesLeft = sitemapRetries + 1;
             while (retriesLeft-- > 0) {
@@ -181,6 +187,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                         if (sitemapResponse.body === null) {
                             break;
                         }
+                        const { fileTypeStream } = await import('file-type');
                         const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
                         if (streamWithType.fileType !== undefined) {
                             contentType = streamWithType.fileType.mime;
@@ -194,7 +201,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                                 sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
                             }
                         }
-                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
+                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
                             if (e !== undefined && e !== null) {
                                 error = { type: 'parser', error: e };
                             }
@@ -222,7 +229,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             }
         }
         else if (source.type === 'raw') {
-            items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
+            items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
                 if (error !== undefined) {
                     logger?.warning(`Malformed sitemap content: ${error}`);
                 }
@@ -231,18 +238,39 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
         if (items === null) {
             continue;
         }
+        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+        let droppedUrlEntries = 0;
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
                 if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
                     logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
                     continue;
                 }
+                // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+                // parent. Raw string sources have no parent URL, so the check is skipped.
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null };
                 }
             }
             if (item.type === 'url') {
+                // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        droppedUrlEntries++;
+                        logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 yield {
                     ...item,
                     originSitemapUrl: source.type === 'url'
@@ -251,6 +279,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                 };
             }
         }
+        if (droppedUrlEntries > 0 && source.type === 'url') {
+            logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
+        }
     }
 }
 /**
@@ -374,7 +405,8 @@ export async function* discoverValidSitemaps(urls, options = {}) {
                 httpClient,
                 logger,
             });
-            for (const sitemapUrl of robotsFile.getSitemaps()) {
+            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                 if (addSitemapUrl(sitemapUrl)) {
                     yield sitemapUrl;
                 }
@@ -383,7 +415,7 @@ export async function* discoverValidSitemaps(urls, options = {}) {
         catch (err) {
             logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
         }
-        const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
+        const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
         if (sitemapUrl !== undefined) {
             if (addSitemapUrl(sitemapUrl)) {
                 yield sitemapUrl;

package/internals/url.d.ts

@@ -1,4 +1,73 @@
 import type { SearchParams } from '@crawlee/types';
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export declare enum EnqueueStrategy {
+    /**
+     * Matches any URLs found
+     */
+    All = "all",
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameHostname = "same-hostname",
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameDomain = "same-domain",
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    SameOrigin = "same-origin"
+}
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
+    allowed: boolean;
+    reason?: string;
+};
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/internals/url.js

@@ -1,3 +1,122 @@
+import { getDomain } from 'tldts';
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export var EnqueueStrategy;
+(function (EnqueueStrategy) {
+    /**
+     * Matches any URLs found
+     */
+    EnqueueStrategy["All"] = "all";
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameHostname"] = "same-hostname";
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameDomain"] = "same-domain";
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    EnqueueStrategy["SameOrigin"] = "same-origin";
+})(EnqueueStrategy || (EnqueueStrategy = {}));
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export const UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+function toUrl(value) {
+    if (value instanceof URL) {
+        return value;
+    }
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+}
+/** Strip a trailing dot so `example.com.` equals `example.com`. */
+function normalizeHostname(hostname) {
+    return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
+}
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export function matchesEnqueueStrategy(strategy, target, origin) {
+    switch (strategy) {
+        case 'all':
+            return true;
+        case 'same-hostname':
+            return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
+        case 'same-domain': {
+            const originDomain = getDomain(origin.hostname, { mixedInputs: false });
+            if (originDomain) {
+                return originDomain === getDomain(target.hostname, { mixedInputs: false });
+            }
+            // No registrable domain (e.g. an IP address), fall back to comparing origins.
+            return target.origin === origin.origin;
+        }
+        case 'same-origin':
+            // Compare scheme/host/port directly so a trailing-dot host is normalized.
+            return (target.protocol === origin.protocol &&
+                normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
+                target.port === origin.port);
+        default:
+            throw new Error(`Unknown enqueue strategy '${strategy}'.`);
+    }
+}
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export function filterUrl(target, origin, strategy) {
+    const targetUrl = toUrl(target);
+    if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
+        return { allowed: false, reason: UNSUPPORTED_SCHEME_MESSAGE };
+    }
+    const originUrl = toUrl(origin);
+    if (originUrl === null) {
+        return { allowed: false, reason: 'invalid origin URL' };
+    }
+    if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
+        return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
+    }
+    return { allowed: true };
+}
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/internals/validation.d.ts

@@ -0,0 +1,25 @@
+import type { z } from 'zod';
+/**
+ * Thrown when an argument fails schema validation.
+ *
+ * Its `message` is a human-readable sentence naming the offending field and the
+ * value it received (rather than a raw JSON dump). The structured
+ * {@link https://zod.dev | zod} issues are available on `issues`, and the
+ * original `ZodError` on `cause`, for programmatic inspection.
+ */
+export declare class ArgumentValidationError extends Error {
+    /** Structured issues from the underlying schema check. */
+    readonly issues: z.ZodError['issues'];
+    /** The raw zod error that triggered this. */
+    readonly cause: z.ZodError;
+    constructor(error: z.ZodError, value: unknown, label?: string);
+}
+/**
+ * Parses `value` with `schema`, returning the typed result (with schema defaults applied).
+ * Throws {@link ArgumentValidationError} on failure.
+ *
+ * The optional `label` names the interface being validated and is appended to every error line
+ * (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
+ * @internal
+ */
+export declare function parseArgument<TValue, TSchema extends z.ZodType>(value: TValue, schema: TSchema, label?: string): TValue & z.output<TSchema>;

package/internals/validation.js

@@ -0,0 +1,140 @@
+/** Formats a zod issue path like `groups[0]` or `countryCode`. */
+function formatIssuePath(path) {
+    let out = '';
+    for (const key of path) {
+        if (typeof key === 'number')
+            out += `[${key}]`;
+        else
+            out += out ? `.${String(key)}` : String(key);
+    }
+    return out;
+}
+/** Reads the value at `path` from the validated input, to include in the error. */
+function valueAtPath(root, path) {
+    let current = root;
+    for (const key of path) {
+        if (current === null || typeof current !== 'object')
+            return undefined;
+        current = current[key];
+    }
+    return current;
+}
+/** Names the runtime type of `value` the way zod's own messages do (`null`, `array`, `string`, …). */
+function describeType(value) {
+    if (value === null)
+        return 'null';
+    if (Array.isArray(value))
+        return 'array';
+    return typeof value;
+}
+/** The bare custom-schema messages that stop at the expected type, e.g. `Invalid input: expected number`. */
+const BARE_EXPECTED_TYPE_MESSAGE = /^Invalid input: expected (an array of .+|a typed array|an object|object|array|function|number|string|boolean)$/;
+/** Longest received string rendered in an error; the rest is elided. */
+const MAX_RENDERED_STRING_LENGTH = 200;
+/** Renders a primitive received value for an error; skips objects/Dates (noisy). */
+function describeReceived(value) {
+    switch (typeof value) {
+        case 'string':
+            // An empty string would render as bare backticks — make it visible.
+            if (value === '')
+                return "''";
+            return value.length > MAX_RENDERED_STRING_LENGTH
+                ? `${value.slice(0, MAX_RENDERED_STRING_LENGTH)}… (${value.length - MAX_RENDERED_STRING_LENGTH} more characters)`
+                : value;
+        case 'number':
+        case 'boolean':
+        case 'bigint':
+            return String(value);
+        default:
+            return undefined;
+    }
+}
+/** Renders the received side of a sentence: ``received the string `abc` ``, `received NaN`, `received array`. */
+function describeReceivedClause(value) {
+    if (typeof value === 'number' && Number.isNaN(value))
+        return 'received NaN';
+    if (value === '')
+        return 'received an empty string';
+    const rendered = describeReceived(value);
+    return rendered === undefined
+        ? `received ${describeType(value)}`
+        : `received the ${describeType(value)} \`${rendered}\``;
+}
+/** Renders one issue as a line each; a union expands into a line per failed arm. */
+function formatIssue(issue, root, basePath) {
+    const path = [...basePath, ...issue.path];
+    // A union's own message is a bare "Invalid input" — the useful part is in `errors`,
+    // whose paths are relative to the union, hence passing `path` down as the base.
+    if (issue.code === 'invalid_union') {
+        return issue.errors.flatMap((arm) => arm.flatMap((nested) => formatIssue(nested, root, path)));
+    }
+    const location = path.length ? ` at \`${formatIssuePath(path)}\`` : '';
+    const value = valueAtPath(root, path);
+    const rendered = describeReceived(value);
+    // ow named the received type ("expected `number` but received type `string`"). The received value is
+    // folded into that clause (``received the string `3` ``) rather than dangling after the location: our
+    // custom schemas stop at the expected type, so the clause is appended; zod's built-in messages already
+    // end with `, received <type>`, so that tail is replaced with the enriched one.
+    let { message } = issue;
+    let got = '';
+    const bareExpected = BARE_EXPECTED_TYPE_MESSAGE.exec(message);
+    const zodReceived = /, received (\S+)$/.exec(message);
+    // `arrayOf` messages name the element type — their expected runtime type is `array`.
+    const expectedType = bareExpected?.[1].startsWith('an array of') ? 'array' : bareExpected?.[1];
+    if (bareExpected && expectedType !== (Number.isNaN(value) ? 'NaN' : describeType(value))) {
+        message += `, ${describeReceivedClause(value)}`;
+    }
+    else if (zodReceived && zodReceived[1] === describeType(value) && rendered !== undefined) {
+        message = `${message.slice(0, zodReceived.index)}, ${describeReceivedClause(value)}`;
+    }
+    else if (rendered !== undefined && !message.endsWith(`received ${rendered}`)) {
+        // Messages that never name a received type (regex, min/max, enums) keep the plain value suffix.
+        got = `, got \`${rendered}\``;
+    }
+    return [`${message}${location}${got}`];
+}
+/**
+ * Formats a `ZodError` as a plain, human-readable message that names the
+ * offending field *and* the value it received (e.g. ``must match pattern
+ * /^[A-Z]{2}$/ at `countryCode`, got `CZE` ``) — closer to the old `ow` errors
+ * than zod's default, which omits the received value.
+ */
+function formatZodError(error, root, label) {
+    const lines = error.issues.flatMap((issue) => formatIssue(issue, root, []));
+    // The label names the validated interface, the way ow's errors ended with "in object `X`".
+    return (label ? lines.map((line) => `${line} in \`${label}\``) : lines).join('\n');
+}
+/**
+ * Thrown when an argument fails schema validation.
+ *
+ * Its `message` is a human-readable sentence naming the offending field and the
+ * value it received (rather than a raw JSON dump). The structured
+ * {@link https://zod.dev | zod} issues are available on `issues`, and the
+ * original `ZodError` on `cause`, for programmatic inspection.
+ */
+export class ArgumentValidationError extends Error {
+    /** Structured issues from the underlying schema check. */
+    issues;
+    /** The raw zod error that triggered this. */
+    cause;
+    constructor(error, value, label) {
+        super(formatZodError(error, value, label), { cause: error });
+        this.name = 'ArgumentValidationError';
+        this.issues = error.issues;
+        this.cause = error;
+    }
+}
+/**
+ * Parses `value` with `schema`, returning the typed result (with schema defaults applied).
+ * Throws {@link ArgumentValidationError} on failure.
+ *
+ * The optional `label` names the interface being validated and is appended to every error line
+ * (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
+ * @internal
+ */
+export function parseArgument(value, schema, label) {
+    const result = schema.safeParse(value);
+    if (!result.success)
+        throw new ArgumentValidationError(result.error, value, label);
+    return result.data;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.98",
+    "version": "4.0.0-rc.0",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -8,6 +8,7 @@
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -42,17 +43,18 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.98",
-        "@crawlee/types": "4.0.0-beta.98",
+        "@crawlee/http-client": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.0",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
         "file-type": "^21.0.0",
-        "ow": "^2.0.0",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
+        "tldts": "^7.0.6",
         "tslib": "^2.8.1",
-        "whatwg-mimetype": "^4.0.0"
+        "whatwg-mimetype": "^4.0.0",
+        "zod": "^4.4.3"
     },
     "lerna": {
         "command": {
@@ -61,5 +63,5 @@
             }
         }
     },
-    "gitHead": "3b8cd86b13e253ab5fc71e631e12a68f7465cee5"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }