@crawlee/utils 3.17.1-beta.9 → 3.18.1-beta.0

package/index.mjs

@@ -2,11 +2,13 @@ import mod from "./index.js";
 
 export default mod;
 export const CLOUDFLARE_RETRY_CSS_SELECTORS = mod.CLOUDFLARE_RETRY_CSS_SELECTORS;
+export const EnqueueStrategy = mod.EnqueueStrategy;
 export const RETRY_CSS_SELECTORS = mod.RETRY_CSS_SELECTORS;
 export const ROTATE_PROXY_ERRORS = mod.ROTATE_PROXY_ERRORS;
 export const RobotsFile = mod.RobotsFile;
 export const RobotsTxtFile = mod.RobotsTxtFile;
 export const Sitemap = mod.Sitemap;
+export const UNSUPPORTED_SCHEME_MESSAGE = mod.UNSUPPORTED_SCHEME_MESSAGE;
 export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
 export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
 export const applySearchParams = mod.applySearchParams;
@@ -20,6 +22,7 @@ export const entries = mod.entries;
 export const expandShadowRoots = mod.expandShadowRoots;
 export const extractUrls = mod.extractUrls;
 export const extractUrlsFromCheerio = mod.extractUrlsFromCheerio;
+export const filterUrl = mod.filterUrl;
 export const getCgroupsVersion = mod.getCgroupsVersion;
 export const getCurrentCpuTicksV2 = mod.getCurrentCpuTicksV2;
 export const getMemoryInfo = mod.getMemoryInfo;
@@ -33,6 +36,7 @@ export const isDocker = mod.isDocker;
 export const isIterable = mod.isIterable;
 export const isLambda = mod.isLambda;
 export const keys = mod.keys;
+export const matchesEnqueueStrategy = mod.matchesEnqueueStrategy;
 export const mergeAsyncIterables = mod.mergeAsyncIterables;
 export const parseOpenGraph = mod.parseOpenGraph;
 export const parseSitemap = mod.parseSitemap;

package/internals/general.js

@@ -16,12 +16,12 @@ const promises_2 = require("node:timers/promises");
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
  */
-exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+exports.URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
 /**
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
-exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
+exports.URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
 let isDockerPromiseCache;
 async function createIsDockerPromise() {
     const promise1 = promises_1.default

package/internals/open_graph_parser.js

@@ -258,32 +258,32 @@ const OPEN_GRAPH_PROPERTIES = [
         outputName: 'articleInfo',
         children: [
             {
-                name: 'music:published_time',
+                name: 'article:published_time',
                 outputName: 'publishedTime',
                 children: [],
             },
             {
-                name: 'music:modified_time',
+                name: 'article:modified_time',
                 outputName: 'modifiedTime',
                 children: [],
             },
             {
-                name: 'music:expiration_time',
+                name: 'article:expiration_time',
                 outputName: 'expirationTime',
                 children: [],
             },
             {
-                name: 'music:author',
+                name: 'article:author',
                 outputName: 'author',
                 children: [],
             },
             {
-                name: 'music:section',
+                name: 'article:section',
                 outputName: 'section',
                 children: [],
             },
             {
-                name: 'music:tag',
+                name: 'article:tag',
                 outputName: 'tag',
                 children: [],
             },

package/internals/robots.d.ts

@@ -1,4 +1,13 @@
 import { Sitemap } from './sitemap';
+import { type EnqueueStrategy } from './url';
+export interface RobotsTxtFileSitemapsOptions {
+    /**
+     * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+     * are always dropped. Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -18,6 +27,7 @@ import { Sitemap } from './sitemap';
  * ```
  */
 export declare class RobotsTxtFile {
+    private url;
     private robots;
     private proxyUrl?;
     private constructor();
@@ -51,16 +61,20 @@ export declare class RobotsTxtFile {
      */
     isAllowed(url: string, userAgent?: string): boolean;
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps(): string[];
+    getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    parseSitemaps(): Promise<Sitemap>;
+    parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    parseUrlsFromSitemaps(): Promise<string[]>;
+    parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
 }
 export { RobotsTxtFile as RobotsFile };

package/internals/robots.js

@@ -3,8 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.RobotsFile = exports.RobotsTxtFile = void 0;
 const tslib_1 = require("tslib");
 const robots_parser_1 = tslib_1.__importDefault(require("robots-parser"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const gotScraping_1 = require("./gotScraping");
 const sitemap_1 = require("./sitemap");
+const url_1 = require("./url");
 let HTTPError;
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
@@ -25,7 +27,13 @@ let HTTPError;
  * ```
  */
 class RobotsTxtFile {
-    constructor(robots, proxyUrl) {
+    constructor(url, robots, proxyUrl) {
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: url
+        });
         Object.defineProperty(this, "robots", {
             enumerable: true,
             configurable: true,
@@ -60,7 +68,7 @@ class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url, content, proxyUrl) {
-        return new RobotsTxtFile((0, robots_parser_1.default)(url, content), proxyUrl);
+        return new RobotsTxtFile(url, (0, robots_parser_1.default)(url, content), proxyUrl);
     }
     static async load(url, proxyUrl, options) {
         if (!HTTPError) {
@@ -75,11 +83,11 @@ class RobotsTxtFile {
                 signal: options?.signal,
                 ...(options?.timeoutMillis ? { timeout: { request: options.timeoutMillis } } : {}),
             });
-            return new RobotsTxtFile((0, robots_parser_1.default)(url.toString(), response.body), proxyUrl);
+            return new RobotsTxtFile(url, (0, robots_parser_1.default)(url.toString(), response.body), proxyUrl);
         }
         catch (e) {
             if (e instanceof HTTPError && e.response.statusCode === 404) {
-                return new RobotsTxtFile({
+                return new RobotsTxtFile(url, {
                     isAllowed() {
                         return true;
                     },
@@ -100,22 +108,37 @@ class RobotsTxtFile {
         return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
     }
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps() {
-        return this.robots.getSitemaps();
+    getSitemaps(options = {}) {
+        const { enqueueStrategy = 'same-hostname' } = options;
+        const sitemaps = [];
+        for (const sitemapUrl of this.robots.getSitemaps()) {
+            // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+            const { allowed, reason } = (0, url_1.filterUrl)(sitemapUrl, this.url, enqueueStrategy);
+            if (!allowed) {
+                log_1.default.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);
+                continue;
+            }
+            sitemaps.push(sitemapUrl);
+        }
+        return sitemaps;
     }
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    async parseSitemaps() {
-        return sitemap_1.Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
+    async parseSitemaps(options = {}) {
+        return sitemap_1.Sitemap.load(this.getSitemaps(options), this.proxyUrl, options);
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    async parseUrlsFromSitemaps() {
-        return (await this.parseSitemaps()).urls;
+    async parseUrlsFromSitemaps(options = {}) {
+        return (await this.parseSitemaps(options)).urls;
     }
 }
 exports.RobotsTxtFile = RobotsTxtFile;

package/internals/sitemap.d.ts

@@ -1,5 +1,6 @@
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Delays } from 'got-scraping';
+import { type EnqueueStrategy } from './url';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -51,6 +52,13 @@ export interface ParseSitemapOptions {
      * If not provided, all nested sitemaps are followed.
      */
     nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+     * sources (no parent URL). Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
 }
 export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
 /**

package/internals/sitemap.js

@@ -13,6 +13,7 @@ const whatwg_mimetype_1 = tslib_1.__importDefault(require("whatwg-mimetype"));
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const iterables_1 = require("./iterables");
 const robots_1 = require("./robots");
+const url_1 = require("./url");
 class SitemapTxtParser extends node_stream_1.Transform {
     constructor() {
         super({
@@ -134,8 +135,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
             this.currentTag = undefined;
         }
-        if (name === 'url' && this.url.loc !== undefined) {
-            this.push({ type: 'url', ...this.url, loc: this.url.loc });
+        if (name === 'url') {
+            if (this.url.loc !== undefined) {
+                this.push({ type: 'url', ...this.url, loc: this.url.loc });
+            }
             this.url = {};
         }
     }
@@ -151,7 +154,10 @@ class SitemapXmlParser extends node_stream_1.Transform {
         }
         text = text.trim();
         if (this.currentTag === 'lastmod') {
-            this.url.lastmod = new Date(text);
+            const lastmod = new Date(text);
+            if (!Number.isNaN(lastmod.getTime())) {
+                this.url.lastmod = lastmod;
+            }
         }
         if (this.currentTag === 'priority') {
             this.url.priority = Number(text);
@@ -166,7 +172,7 @@ class SitemapXmlParser extends node_stream_1.Transform {
 async function* parseSitemap(initialSources, proxyUrl, options) {
     const { gotScraping } = await import('got-scraping');
     const { fileTypeStream } = await import('file-type');
-    const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts, reportNetworkErrors = true, nestedSitemapFilter, } = options ?? {};
+    const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
     const createParser = (contentType = '', url) => {
@@ -192,8 +198,10 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
             continue;
         }
         let items = null;
+        // Parent URL, parsed once and reused as the origin for the strategy checks below.
+        let sitemapUrl;
         if (source.type === 'url') {
-            const sitemapUrl = new URL(source.url);
+            sitemapUrl = new URL(source.url);
             visitedSitemapUrls.add(sitemapUrl.toString());
             let retriesLeft = sitemapRetries + 1;
             while (retriesLeft-- > 0) {
@@ -264,18 +272,39 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
         if (items === null) {
             continue;
         }
+        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+        let droppedUrlEntries = 0;
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
                 if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
                     log_1.default.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
                     continue;
                 }
+                // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+                // parent. Raw string sources have no parent URL, so the check is skipped.
+                if (source.type === 'url') {
+                    const { allowed, reason } = (0, url_1.filterUrl)(item.url, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        log_1.default.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null };
                 }
             }
             if (item.type === 'url') {
+                // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+                if (source.type === 'url') {
+                    const { allowed, reason } = (0, url_1.filterUrl)(item.loc, sitemapUrl, enqueueStrategy);
+                    if (!allowed) {
+                        droppedUrlEntries++;
+                        log_1.default.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
                 yield {
                     ...item,
                     originSitemapUrl: source.type === 'url'
@@ -284,6 +313,9 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
                 };
             }
         }
+        if (droppedUrlEntries > 0 && source.type === 'url') {
+            log_1.default.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
+        }
     }
 }
 /**
@@ -406,7 +438,8 @@ async function* discoverValidSitemaps(urls, options = {}) {
                 timeoutMillis: requestTimeoutMillis,
                 signal,
             });
-            for (const sitemapUrl of robotsFile.getSitemaps()) {
+            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                 if (addSitemapUrl(sitemapUrl)) {
                     yield sitemapUrl;
                 }
@@ -415,7 +448,7 @@ async function* discoverValidSitemaps(urls, options = {}) {
         catch (err) {
             log_1.default.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
         }
-        const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
+        const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
         if (sitemapUrl !== undefined) {
             if (addSitemapUrl(sitemapUrl)) {
                 yield sitemapUrl;

package/internals/social.js

@@ -10,7 +10,10 @@ const tslib_1 = require("tslib");
 const cheerio = tslib_1.__importStar(require("cheerio"));
 const cheerio_1 = require("./cheerio");
 // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
-const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
+// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
+// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
+// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
+const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
 /**
  * Regular expression to exactly match a single email address.
  * It has the following form: `/^...$/i`.

package/internals/systemInfoV2/cpu-info.js

@@ -51,6 +51,7 @@ function getCurrentCpuTicks() {
     }, { idle: 0, total: 0 });
     const idleTicksDelta = ticks.idle - previousTicks.idle;
     const totalTicksDelta = ticks.total - previousTicks.total;
+    Object.assign(previousTicks, ticks);
     return totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
 }
 /**

package/internals/url.d.ts

@@ -1,4 +1,73 @@
 export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export declare enum EnqueueStrategy {
+    /**
+     * Matches any URLs found
+     */
+    All = "all",
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameHostname = "same-hostname",
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    SameDomain = "same-domain",
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    SameOrigin = "same-origin"
+}
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
+    allowed: boolean;
+    reason?: string;
+};
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/internals/url.js

@@ -1,6 +1,128 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNSUPPORTED_SCHEME_MESSAGE = exports.EnqueueStrategy = void 0;
+exports.matchesEnqueueStrategy = matchesEnqueueStrategy;
+exports.filterUrl = filterUrl;
 exports.applySearchParams = applySearchParams;
+const tldts_1 = require("tldts");
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+var EnqueueStrategy;
+(function (EnqueueStrategy) {
+    /**
+     * Matches any URLs found
+     */
+    EnqueueStrategy["All"] = "all";
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameHostname"] = "same-hostname";
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameDomain"] = "same-domain";
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    EnqueueStrategy["SameOrigin"] = "same-origin";
+})(EnqueueStrategy || (exports.EnqueueStrategy = EnqueueStrategy = {}));
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+exports.UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+function toUrl(value) {
+    if (value instanceof URL) {
+        return value;
+    }
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+}
+/** Strip a trailing dot so `example.com.` equals `example.com`. */
+function normalizeHostname(hostname) {
+    return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
+}
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+function matchesEnqueueStrategy(strategy, target, origin) {
+    switch (strategy) {
+        case 'all':
+            return true;
+        case 'same-hostname':
+            return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
+        case 'same-domain': {
+            const originDomain = (0, tldts_1.getDomain)(origin.hostname, { mixedInputs: false });
+            if (originDomain) {
+                return originDomain === (0, tldts_1.getDomain)(target.hostname, { mixedInputs: false });
+            }
+            // No registrable domain (e.g. an IP address), fall back to comparing origins.
+            return target.origin === origin.origin;
+        }
+        case 'same-origin':
+            // Compare scheme/host/port directly so a trailing-dot host is normalized.
+            return (target.protocol === origin.protocol &&
+                normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
+                target.port === origin.port);
+        default:
+            throw new Error(`Unknown enqueue strategy '${strategy}'.`);
+    }
+}
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+function filterUrl(target, origin, strategy) {
+    const targetUrl = toUrl(target);
+    if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
+        return { allowed: false, reason: exports.UNSUPPORTED_SCHEME_MESSAGE };
+    }
+    const originUrl = toUrl(origin);
+    if (originUrl === null) {
+        return { allowed: false, reason: 'invalid origin URL' };
+    }
+    if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
+        return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
+    }
+    return { allowed: true };
+}
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "3.17.1-beta.9",
+    "version": "3.18.1-beta.0",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=16.0.0"
@@ -49,7 +49,7 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/types": "3.17.1-beta.9",
+        "@crawlee/types": "3.18.1-beta.0",
         "@types/sax": "^1.2.7",
         "cheerio": "1.0.0-rc.12",
         "file-type": "^21.3.1",
@@ -57,6 +57,7 @@
         "ow": "^0.28.1",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
+        "tldts": "^7.0.0",
         "tslib": "^2.4.0",
         "whatwg-mimetype": "^4.0.0"
     },
@@ -70,5 +71,5 @@
             }
         }
     },
-    "gitHead": "c547ce4489d8617620a51516cc885187503f3136"
+    "gitHead": "fe3605157b4d0262d09898aa18626ab5a3e03702"
 }