@crawlee/utils 4.0.0-beta.126 → 4.0.0-beta.128

package/index.d.ts

@@ -1,5 +1,6 @@
 export { htmlToText } from './internals/cheerio.js';
 export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
 export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
 export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';

package/index.js

@@ -1,5 +1,6 @@
 export { htmlToText } from './internals/cheerio.js';
 export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
 export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
 export * from './internals/open_graph_parser.js';

package/internals/robots.d.ts

@@ -1,6 +1,15 @@
 import type { BaseHttpClient } from '@crawlee/http-client';
 import type { CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
+import { type EnqueueStrategy } from './url.js';
+export interface RobotsTxtFileSitemapsOptions {
+    /**
+     * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+     * are always dropped. Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -57,15 +66,19 @@ export declare class RobotsTxtFile {
      */
     isAllowed(url: string, userAgent?: string): boolean;
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps(): string[];
+    getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    parseSitemaps(): Promise<Sitemap>;
+    parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    parseUrlsFromSitemaps(): Promise<string[]>;
+    parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
 }

package/internals/robots.js

@@ -1,6 +1,7 @@
 import { FetchHttpClient } from '@crawlee/http-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
+import { filterUrl } from './url.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -20,10 +21,12 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export class RobotsTxtFile {
+    #url;
     #robots;
     #proxyUrl;
     #logger;
-    constructor(robots, proxyUrl, logger) {
+    constructor(url, robots, proxyUrl, logger) {
+        this.#url = url;
         this.#robots = robots;
         this.#proxyUrl = proxyUrl;
         this.#logger = logger;
@@ -50,7 +53,7 @@ export class RobotsTxtFile {
      */
     static from(url, content, proxyUrl) {
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
+        return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
     static async load(url, options) {
         const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
@@ -63,7 +66,7 @@ export class RobotsTxtFile {
             throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
         }
         if (response.status === 404) {
-            return new RobotsTxtFile({
+            return new RobotsTxtFile(url, {
                 isAllowed() {
                     return true;
                 },
@@ -76,7 +79,7 @@ export class RobotsTxtFile {
             }, proxyUrl, logger);
         }
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
+        return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }
     /**
      * Get crawl delay for a given user agent.
@@ -94,21 +97,36 @@ export class RobotsTxtFile {
         return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
     }
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps() {
-        return this.#robots.getSitemaps();
+    getSitemaps(options = {}) {
+        const { enqueueStrategy = 'same-hostname' } = options;
+        const sitemaps = [];
+        for (const sitemapUrl of this.#robots.getSitemaps()) {
+            // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+            const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
+            if (!allowed) {
+                this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
+                continue;
+            }
+            sitemaps.push(sitemapUrl);
+        }
+        return sitemaps;
     }
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    async parseSitemaps() {
-        return Sitemap.load(this.#robots.getSitemaps(), this.#proxyUrl, { logger: this.#logger });
+    async parseSitemaps(options = {}) {
+        return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    async parseUrlsFromSitemaps() {
-        return (await this.parseSitemaps()).urls;
+    async parseUrlsFromSitemaps(options = {}) {
+        return (await this.parseSitemaps(options)).urls;
     }
 }

package/internals/sitemap.d.ts

@@ -1,5 +1,6 @@
 import type { BaseHttpClient } from '@crawlee/http-client';
 import type { CrawleeLogger } from '@crawlee/types';
+import { type EnqueueStrategy } from './url.js';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -55,6 +56,13 @@ export interface ParseSitemapOptions {
      * If not provided, all nested sitemaps are followed.
      */
     nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+     * sources (no parent URL). Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Optional logger for reporting warnings during sitemap parsing.
      */

package/internals/sitemap.js

@@ -6,6 +6,7 @@ import { FetchHttpClient } from '@crawlee/http-client';
 import MIMEType from 'whatwg-mimetype';
 import { mergeAsyncIterables } from './iterables.js';
 import { RobotsTxtFile } from './robots.js';
+import { filterUrl } from './url.js';
 class SitemapTxtParser extends Transform {
     #decoder = new StringDecoder('utf8');
     #buffer = '';
@@ -132,7 +133,7 @@ class SitemapXmlParser extends Transform {
     }
 }
 export async function* parseSitemap(initialSources, proxyUrl, options) {
-    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
+    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
     const createParser = async (contentType = '', url) => {
@@ -157,8 +158,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             continue;
         }
         let items = null;
+        // Parent URL, parsed once and reused as the origin for the strategy checks below.
+        let sitemapUrl;
         if (source.type === 'url') {
-            const sitemapUrl = new URL(source.url);
+            sitemapUrl = new URL(source.url);
             visitedSitemapUrls.add(sitemapUrl.toString());
             let retriesLeft = sitemapRetries + 1;
             while (retriesLeft-- > 0) {
@@ -235,18 +238,39 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
         if (items === null) {
             continue;
         }
+        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+        let droppedUrlEntries = 0;
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
                 if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
                     logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
                     continue;
                 }
+                // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+                // parent. Raw string sources have no parent URL, so the check is skipped.
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null };
                 }
             }
             if (item.type === 'url') {
+                // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        droppedUrlEntries++;
+                        logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 yield {
                     ...item,
                     originSitemapUrl: source.type === 'url'
@@ -255,6 +279,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                 };
             }
         }
+        if (droppedUrlEntries > 0 && source.type === 'url') {
+            logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
+        }
     }
 }
 /**
@@ -378,7 +405,8 @@ export async function* discoverValidSitemaps(urls, options = {}) {
                 httpClient,
                 logger,
             });
-            for (const sitemapUrl of robotsFile.getSitemaps()) {
+            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                 if (addSitemapUrl(sitemapUrl)) {
                     yield sitemapUrl;
                 }
@@ -387,7 +415,7 @@ export async function* discoverValidSitemaps(urls, options = {}) {
         catch (err) {
             logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
         }
-        const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
+        const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
         if (sitemapUrl !== undefined) {
             if (addSitemapUrl(sitemapUrl)) {
                 yield sitemapUrl;

package/internals/url.d.ts

@@ -1,4 +1,73 @@
 import type { SearchParams } from '@crawlee/types';
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export declare enum EnqueueStrategy {
+    /**
+     * Matches any URLs found
+     */
+    All = "all",
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameHostname = "same-hostname",
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameDomain = "same-domain",
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    SameOrigin = "same-origin"
+}
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
+    allowed: boolean;
+    reason?: string;
+};
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/internals/url.js

@@ -1,3 +1,122 @@
+import { getDomain } from 'tldts';
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export var EnqueueStrategy;
+(function (EnqueueStrategy) {
+    /**
+     * Matches any URLs found
+     */
+    EnqueueStrategy["All"] = "all";
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameHostname"] = "same-hostname";
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameDomain"] = "same-domain";
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    EnqueueStrategy["SameOrigin"] = "same-origin";
+})(EnqueueStrategy || (EnqueueStrategy = {}));
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export const UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+function toUrl(value) {
+    if (value instanceof URL) {
+        return value;
+    }
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+}
+/** Strip a trailing dot so `example.com.` equals `example.com`. */
+function normalizeHostname(hostname) {
+    return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
+}
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export function matchesEnqueueStrategy(strategy, target, origin) {
+    switch (strategy) {
+        case 'all':
+            return true;
+        case 'same-hostname':
+            return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
+        case 'same-domain': {
+            const originDomain = getDomain(origin.hostname, { mixedInputs: false });
+            if (originDomain) {
+                return originDomain === getDomain(target.hostname, { mixedInputs: false });
+            }
+            // No registrable domain (e.g. an IP address), fall back to comparing origins.
+            return target.origin === origin.origin;
+        }
+        case 'same-origin':
+            // Compare scheme/host/port directly so a trailing-dot host is normalized.
+            return (target.protocol === origin.protocol &&
+                normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
+                target.port === origin.port);
+        default:
+            throw new Error(`Unknown enqueue strategy '${strategy}'.`);
+    }
+}
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export function filterUrl(target, origin, strategy) {
+    const targetUrl = toUrl(target);
+    if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
+        return { allowed: false, reason: UNSUPPORTED_SCHEME_MESSAGE };
+    }
+    const originUrl = toUrl(origin);
+    if (originUrl === null) {
+        return { allowed: false, reason: 'invalid origin URL' };
+    }
+    if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
+        return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
+    }
+    return { allowed: true };
+}
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.126",
+    "version": "4.0.0-beta.128",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -43,14 +43,15 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.126",
-        "@crawlee/types": "4.0.0-beta.126",
+        "@crawlee/http-client": "4.0.0-beta.128",
+        "@crawlee/types": "4.0.0-beta.128",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
         "file-type": "^21.0.0",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
+        "tldts": "^7.0.6",
         "tslib": "^2.8.1",
         "whatwg-mimetype": "^4.0.0",
         "zod": "^4.4.3"
@@ -62,5 +63,5 @@
             }
         }
     },
-    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
+    "gitHead": "2c017f04d564e8fa13855bf19f31f43cb65f4f44"
 }