@crawlee/utils 4.0.0-beta.23 → 4.0.0-beta.231

package/internals/schemas.js

@@ -0,0 +1,115 @@
+import { BaseHttpClient } from '@crawlee/http-client';
+import { z } from 'zod';
+/**
+ * Accepts any object (including arrays and functions).
+ * @internal
+ */
+export const anyObject = z.custom((value) => (typeof value === 'object' && value !== null) || typeof value === 'function', { message: 'Invalid input: expected object' });
+/**
+ * Accepts any array without validating its items (cheap for huge arrays).
+ * @internal
+ */
+export const anyArray = z.custom(Array.isArray, { message: 'Invalid input: expected array' });
+/**
+ * Accepts any function.
+ * @internal
+ */
+export const anyFunction = z.custom((value) => typeof value === 'function', {
+    message: 'Invalid input: expected function',
+});
+/**
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
+ * @internal
+ */
+export const anyNumber = z.custom((value) => typeof value === 'number' && !Number.isNaN(value), {
+    message: 'Invalid input: expected number',
+});
+/**
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
+ * @internal
+ */
+export function objectWithKeys(keys, message) {
+    return z.custom((value) => ((typeof value === 'object' && value !== null) || typeof value === 'function') &&
+        keys.every((key) => key in value), {
+        message: message ?? `Invalid input: expected an object with keys ${keys.map((key) => `'${key}'`).join(', ')}`,
+    });
+}
+/**
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
+ * @internal
+ */
+export const httpClient = z.instanceof(BaseHttpClient);
+/**
+ * Accepts any object implementing the CrawleeLogger interface.
+ * @internal
+ */
+export const logger = objectWithKeys(['child', 'info', 'error', 'warning'], "Expected an object implementing the CrawleeLogger interface (missing one of 'child', 'info', 'error', 'warning'), got something else.");
+/**
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
+ * @internal
+ */
+export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(value instanceof DataView), { message: 'Invalid input: expected a typed array' });
+/**
+ * Accepts any non-null, non-array object.
+ * @internal
+ */
+export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
+const storageRequestShape = z.looseObject({
+    id: z.string(),
+    url: z.url({ protocol: /^https?$/ }),
+    uniqueKey: z.string(),
+    method: z.string().optional(),
+    retryCount: z.number().int().optional(),
+    handledAt: z.union([z.string(), z.date()]).optional(),
+});
+/**
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
+ * @internal
+ */
+export const storageRequest = z.compile(storageRequestShape);
+/**
+ * {@link storageRequest} before an id is assigned.
+ * @internal
+ */
+export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
+/**
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
+ * @internal
+ */
+export function arrayOf(item, elements) {
+    return z.array(item, {
+        error: (issue) => issue.code === 'invalid_type' ? `Invalid input: expected an array of ${elements}` : undefined,
+    });
+}
+/**
+ * Batch of {@link storageRequestWithoutId}.
+ * @internal
+ */
+export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
+/**
+ * Options of request queue add/update operations.
+ * @internal
+ */
+export const requestQueueOperationOptions = z.object({
+    forefront: z.boolean().optional(),
+});
+/**
+ * Options of key-value store `listKeys`.
+ * @internal
+ */
+export const keyValueStoreListKeysOptions = z.object({
+    prefix: z.string().optional(),
+    exclusiveStartKey: z.string().optional(),
+    limit: z.number().int().gt(0).optional(),
+});
+/**
+ * Options of dataset item listing.
+ * @internal
+ */
+export const datasetListItemsOptions = z.object({
+    desc: z.boolean().optional(),
+    limit: z.number().int().optional(),
+    offset: z.number().int().optional(),
+});

package/internals/sitemap.d.ts

@@ -1,4 +1,6 @@
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
+import { type EnqueueStrategy } from './url.js';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -47,6 +49,24 @@ export interface ParseSitemapOptions {
      * Custom HTTP client to be used for fetching sitemaps.
      */
     httpClient?: BaseHttpClient;
+    /**
+     * Optional filter for nested sitemap URLs discovered in sitemap index files.
+     * Called with the URL of each child sitemap before it is fetched.
+     * Return `true` to include the sitemap, `false` to skip it.
+     * If not provided, all nested sitemaps are followed.
+     */
+    nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+     * sources (no parent URL). Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+    /**
+     * Optional logger for reporting warnings during sitemap parsing.
+     */
+    logger?: CrawleeLogger;
 }
 export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
 /**
@@ -62,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
  * ```
  */
 export declare class Sitemap {
+    #private;
     readonly urls: string[];
     constructor(urls: string[]);
     /**
@@ -83,7 +104,6 @@ export declare class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
-    protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
 }
 /**
  * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
@@ -97,10 +117,31 @@ export declare function discoverValidSitemaps(urls: string[], options?: {
      * Proxy URL to be used for network requests.
      */
     proxyUrl?: string;
+    /**
+     * Timeout in milliseconds for the entire `discoverValidSitemaps` call.
+     * An `AbortController` is created internally and its signal is passed to every HTTP request,
+     * so the whole discovery operation is cancelled once the timeout elapses.
+     * Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
+     */
+    timeoutMillis?: number;
+    /**
+     * An external `AbortSignal` to cancel the entire discovery operation.
+     * If both `signal` and `timeout` are provided, the operation is cancelled
+     * when either the signal is aborted or the timeout elapses (whichever comes first).
+     */
+    signal?: AbortSignal;
+    /**
+     * Timeout in milliseconds for each individual HTTP request during discovery.
+     * Defaults to `20000` ms (20 seconds).
+     */
+    requestTimeoutMillis?: number;
     /**
      * HTTP client to be used for network requests.
      */
     httpClient?: BaseHttpClient;
+    /**
+     * Optional logger for reporting warnings during sitemap discovery.
+     */
+    logger?: CrawleeLogger;
 }): AsyncIterable<string>;
 export {};
-//# sourceMappingURL=sitemap.d.ts.map

package/internals/sitemap.js

@@ -3,32 +3,30 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
 import { StringDecoder } from 'node:string_decoder';
 import { createGunzip } from 'node:zlib';
 import { FetchHttpClient } from '@crawlee/http-client';
-import { fileTypeStream } from 'file-type';
-import sax from 'sax';
-import MIMEType from 'whatwg-mimetype';
-import log from '@apify/log';
+import { MIMEType } from 'whatwg-mimetype';
 import { mergeAsyncIterables } from './iterables.js';
-import { RobotsFile } from './robots.js';
+import { RobotsTxtFile } from './robots.js';
+import { filterUrl } from './url.js';
 class SitemapTxtParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    buffer = '';
+    #decoder = new StringDecoder('utf8');
+    #buffer = '';
     constructor() {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.processBuffer(this.decoder.write(chunk), false);
+                this.#processBuffer(this.#decoder.write(chunk), false);
                 callback();
             },
             flush: (callback) => {
-                this.processBuffer(this.decoder.end(), true);
+                this.#processBuffer(this.#decoder.end(), true);
                 callback();
             },
         });
     }
-    processBuffer(input, finalize) {
-        this.buffer += input;
-        if (finalize || this.buffer.includes('\n')) {
-            const parts = this.buffer
+    #processBuffer(input, finalize) {
+        this.#buffer += input;
+        if (finalize || this.#buffer.includes('\n')) {
+            const parts = this.#buffer
                 .split('\n')
                 .map((part) => part.trim())
                 .filter((part) => part.length > 0);
@@ -36,99 +34,113 @@ class SitemapTxtParser extends Transform {
                 for (const url of parts) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = '';
+                this.#buffer = '';
             }
             else if (parts.length > 0) {
                 for (const url of parts.slice(0, -1)) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = parts.at(-1);
+                this.#buffer = parts.at(-1);
             }
         }
     }
 }
 class SitemapXmlParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    parser = new sax.SAXParser(true);
-    rootTagName;
-    currentTag = undefined;
-    url = {};
-    constructor() {
+    #decoder = new StringDecoder('utf8');
+    #parser;
+    #rootTagName;
+    #currentTag = undefined;
+    #url = {};
+    static async create() {
+        const { SAXParser } = (await import('sax')).default;
+        return new SitemapXmlParser(new SAXParser(true));
+    }
+    constructor(parser) {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.parser.write(this.decoder.write(chunk));
+                this.#parser.write(this.#decoder.write(chunk));
                 callback();
             },
             flush: (callback) => {
-                const rest = this.decoder.end();
+                const rest = this.#decoder.end();
                 if (rest.length > 0) {
-                    this.parser.write(rest);
+                    this.#parser.write(rest);
                 }
-                this.parser.end();
+                this.#parser.end();
                 callback();
             },
         });
-        this.parser.onopentag = this.onOpenTag.bind(this);
-        this.parser.onclosetag = this.onCloseTag.bind(this);
-        this.parser.ontext = this.onText.bind(this);
-        this.parser.oncdata = this.onText.bind(this);
-        this.parser.onerror = this.destroy.bind(this);
+        this.#parser = parser;
+        this.#parser.onopentag = this.#onOpenTag.bind(this);
+        this.#parser.onclosetag = this.#onCloseTag.bind(this);
+        this.#parser.ontext = this.#onText.bind(this);
+        this.#parser.oncdata = this.#onText.bind(this);
+        this.#parser.onerror = this.destroy.bind(this);
     }
-    onOpenTag(node) {
-        if (this.rootTagName !== undefined) {
+    #onOpenTag(node) {
+        if (this.#rootTagName !== undefined) {
             if (node.name === 'loc' ||
                 node.name === 'lastmod' ||
                 node.name === 'priority' ||
                 node.name === 'changefreq') {
-                this.currentTag = node.name;
+                this.#currentTag = node.name;
             }
         }
         if (node.name === 'urlset') {
-            this.rootTagName = 'urlset';
+            this.#rootTagName = 'urlset';
         }
         if (node.name === 'sitemapindex') {
-            this.rootTagName = 'sitemapindex';
+            this.#rootTagName = 'sitemapindex';
         }
     }
-    onCloseTag(name) {
+    #onCloseTag(name) {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
-            this.currentTag = undefined;
+            this.#currentTag = undefined;
         }
-        if (name === 'url' && this.url.loc !== undefined) {
-            this.push({ type: 'url', ...this.url, loc: this.url.loc });
-            this.url = {};
+        if (name === 'url') {
+            if (this.#url.loc !== undefined) {
+                this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
+            }
+            this.#url = {};
         }
     }
-    onText(text) {
-        if (this.currentTag === 'loc') {
-            if (this.rootTagName === 'sitemapindex') {
-                this.push({ type: 'sitemapUrl', url: text.trim() });
+    #onText(text) {
+        text = text.trim();
+        // Whitespace around a CDATA section arrives as separate text events.
+        if (text === '') {
+            return;
+        }
+        if (this.#currentTag === 'loc') {
+            if (this.#rootTagName === 'sitemapindex') {
+                this.push({ type: 'sitemapUrl', url: text });
             }
-            if (this.rootTagName === 'urlset') {
-                this.url ??= {};
-                this.url.loc = text.trim();
+            if (this.#rootTagName === 'urlset') {
+                this.#url ??= {};
+                this.#url.loc = text;
             }
         }
-        text = text.trim();
-        if (this.currentTag === 'lastmod') {
-            this.url.lastmod = new Date(text);
+        if (this.#currentTag === 'lastmod') {
+            const lastmod = new Date(text);
+            if (!Number.isNaN(lastmod.getTime())) {
+                this.#url.lastmod = lastmod;
+            }
         }
-        if (this.currentTag === 'priority') {
-            this.url.priority = Number(text);
+        if (this.#currentTag === 'priority') {
+            this.#url.priority = Number(text);
         }
-        if (this.currentTag === 'changefreq') {
+        if (this.#currentTag === 'changefreq') {
             if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
-                this.url.changefreq = text;
+                this.#url.changefreq = text;
             }
         }
     }
 }
 export async function* parseSitemap(initialSources, proxyUrl, options) {
-    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
+    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
-    const createParser = (contentType = '', url) => {
+    const createParser = async (contentType = '', url) => {
         let mimeType;
         try {
             mimeType = new MIMEType(contentType);
@@ -137,7 +149,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             mimeType = null;
         }
         if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
-            return new SitemapXmlParser();
+            return SitemapXmlParser.create();
         }
         if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
             return new SitemapTxtParser();
@@ -147,12 +159,13 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
     while (sources.length > 0) {
         const source = sources.shift();
         if ((source?.depth ?? 0) > maxDepth) {
-            log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
             continue;
         }
         let items = null;
+        // Parent URL, parsed once and reused as the origin for the strategy checks below.
+        let sitemapUrl;
         if (source.type === 'url') {
-            const sitemapUrl = new URL(source.url);
+            sitemapUrl = new URL(source.url);
             visitedSitemapUrls.add(sitemapUrl.toString());
             let retriesLeft = sitemapRetries + 1;
             while (retriesLeft-- > 0) {
@@ -162,11 +175,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                         sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
                             method: 'GET',
                             headers: {
-                                accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
+                                accept: '*/*',
                             },
                         }), {
                             proxyUrl,
-                            timeout,
+                            timeoutMillis: timeout,
                         });
                     }
                     catch (error) {
@@ -178,6 +191,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                         if (sitemapResponse.body === null) {
                             break;
                         }
+                        const { fileTypeStream } = await import('file-type');
                         const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
                         if (streamWithType.fileType !== undefined) {
                             contentType = streamWithType.fileType.mime;
@@ -191,7 +205,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                                 sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
                             }
                         }
-                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
+                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
                             if (e !== undefined && e !== null) {
                                 error = { type: 'parser', error: e };
                             }
@@ -214,28 +228,53 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                     }
                 }
                 catch (e) {
-                    log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
+                    logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
                 }
             }
         }
         else if (source.type === 'raw') {
-            items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
+            items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
                 if (error !== undefined) {
-                    log.warning(`Malformed sitemap content: ${error}`);
+                    logger?.warning(`Malformed sitemap content: ${error}`);
                 }
             });
         }
         if (items === null) {
             continue;
         }
+        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+        let droppedUrlEntries = 0;
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
+                if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
+                    logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
+                    continue;
+                }
+                // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+                // parent. Raw string sources have no parent URL, so the check is skipped.
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null };
                 }
             }
             if (item.type === 'url') {
+                // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        droppedUrlEntries++;
+                        logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 yield {
                     ...item,
                     originSitemapUrl: source.type === 'url'
@@ -244,6 +283,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                 };
             }
         }
+        if (droppedUrlEntries > 0 && source.type === 'url') {
+            logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
+        }
     }
 }
 /**
@@ -285,7 +327,7 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async load(urls, proxyUrl, parseSitemapOptions) {
-        return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
+        return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
     }
     /**
      * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -293,16 +335,17 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
-        return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
+        return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
     }
-    static async parse(sources, proxyUrl, parseSitemapOptions) {
+    static async #parse(sources, proxyUrl, parseSitemapOptions) {
         const urls = [];
         try {
             for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
                 urls.push(item.loc);
             }
         }
-        catch {
+        catch (e) {
+            parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
             return new Sitemap([]);
         }
         return new Sitemap(urls);
@@ -316,7 +359,19 @@ export class Sitemap {
  * @returns An async iterable with the discovered sitemap URLs.
  */
 export async function* discoverValidSitemaps(urls, options = {}) {
-    const { proxyUrl, httpClient = new FetchHttpClient() } = options;
+    const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
+    const onExternalAbort = () => controller.abort();
+    if (externalSignal) {
+        if (externalSignal.aborted) {
+            controller.abort();
+        }
+        else {
+            externalSignal.addEventListener('abort', onExternalAbort, { once: true });
+        }
+    }
+    const signal = controller.signal;
     const sitemapUrls = new Set();
     const addSitemapUrl = (url) => {
         const sizeBefore = sitemapUrls.size;
@@ -331,7 +386,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
             return false;
         }
         try {
-            const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
+            const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
+                proxyUrl,
+                timeoutMillis: requestTimeoutMillis,
+                signal,
+            });
             return response.status >= 200 && response.status < 400;
         }
         catch {
@@ -343,17 +402,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
             return;
         }
         try {
-            const robotsFile = await RobotsFile.find(domainUrls[0], { proxyUrl, httpClient });
-            for (const sitemapUrl of robotsFile.getSitemaps()) {
+            const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
+                proxyUrl,
+                timeoutMillis: requestTimeoutMillis,
+                signal,
+                httpClient,
+                logger,
+            });
+            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                 if (addSitemapUrl(sitemapUrl)) {
                     yield sitemapUrl;
                 }
             }
         }
         catch (err) {
-            log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
+            logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
         }
-        const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
+        const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
         if (sitemapUrl !== undefined) {
             if (addSitemapUrl(sitemapUrl)) {
                 yield sitemapUrl;
@@ -361,14 +427,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
         }
         else {
             const firstUrl = new URL(domainUrls[0]);
+            // The guessed candidates are well-known paths on the same origin, so the query string
+            // and fragment of the input URL are not part of them - carrying them over probes (and
+            // yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
+            firstUrl.search = '';
+            firstUrl.hash = '';
             const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
-            for (const pathname of possibleSitemapPathnames) {
+            const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
                 firstUrl.pathname = pathname;
-                if (await urlExists(firstUrl.toString())) {
-                    if (addSitemapUrl(firstUrl.toString())) {
-                        yield firstUrl.toString();
+                return firstUrl.toString();
+            });
+            const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
+            for (const [index, result] of candidateResults.entries()) {
+                const candidateSitemapUrl = candidateSitemapUrls[index];
+                if (result.status === 'fulfilled') {
+                    if (result.value && addSitemapUrl(candidateSitemapUrl)) {
+                        yield candidateSitemapUrl;
                     }
                 }
+                else {
+                    logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
+                        error: result.reason,
+                    });
+                }
             }
         }
     };
@@ -380,12 +461,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
     }, {});
     const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
     const discoveredUrls = new Set();
-    for await (const url of mergeAsyncIterables(...iterables)) {
-        if (discoveredUrls.has(url)) {
-            continue;
+    try {
+        for await (const url of mergeAsyncIterables(...iterables)) {
+            if (discoveredUrls.has(url)) {
+                continue;
+            }
+            discoveredUrls.add(url);
+            yield url;
         }
-        discoveredUrls.add(url);
-        yield url;
+    }
+    finally {
+        clearTimeout(timeoutHandle);
+        externalSignal?.removeEventListener('abort', onExternalAbort);
     }
 }
-//# sourceMappingURL=sitemap.js.map