@crawlee/utils 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/url.js

@@ -1,3 +1,122 @@
+import { getDomain } from 'tldts';
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol          Domain
+ * ┌────┐          ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │       └─────────────────┤
+ * │             Hostname    │
+ * │                         │
+ * └─────────────────────────┘
+ *          Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
+export var EnqueueStrategy;
+(function (EnqueueStrategy) {
+    /**
+     * Matches any URLs found
+     */
+    EnqueueStrategy["All"] = "all";
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `https://example.com/hello` will not be matched.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameHostname"] = "same-hostname";
+    /**
+     * Matches any URLs that have the same domain as the base URL.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     *
+     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
+     */
+    EnqueueStrategy["SameDomain"] = "same-domain";
+    /**
+     * Matches any URLs that have the same hostname and protocol.
+     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+     * `http://wow.example.com/hello` will not be matched.
+     *
+     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+     */
+    EnqueueStrategy["SameOrigin"] = "same-origin";
+})(EnqueueStrategy || (EnqueueStrategy = {}));
+/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
+export const UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+function toUrl(value) {
+    if (value instanceof URL) {
+        return value;
+    }
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+}
+/** Strip a trailing dot so `example.com.` equals `example.com`. */
+function normalizeHostname(hostname) {
+    return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
+}
+/**
+ * Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
+ * considered here (use {@link filterUrl} for the combined scheme + strategy check).
+ *
+ * The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
+ * (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
+ */
+export function matchesEnqueueStrategy(strategy, target, origin) {
+    switch (strategy) {
+        case 'all':
+            return true;
+        case 'same-hostname':
+            return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
+        case 'same-domain': {
+            const originDomain = getDomain(origin.hostname, { mixedInputs: false });
+            if (originDomain) {
+                return originDomain === getDomain(target.hostname, { mixedInputs: false });
+            }
+            // No registrable domain (e.g. an IP address), fall back to comparing origins.
+            return target.origin === origin.origin;
+        }
+        case 'same-origin':
+            // Compare scheme/host/port directly so a trailing-dot host is normalized.
+            return (target.protocol === origin.protocol &&
+                normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
+                target.port === origin.port);
+        default:
+            throw new Error(`Unknown enqueue strategy '${strategy}'.`);
+    }
+}
+/**
+ * Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
+ * scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
+ */
+export function filterUrl(target, origin, strategy) {
+    const targetUrl = toUrl(target);
+    if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
+        return { allowed: false, reason: UNSUPPORTED_SCHEME_MESSAGE };
+    }
+    const originUrl = toUrl(origin);
+    if (originUrl === null) {
+        return { allowed: false, reason: 'invalid origin URL' };
+    }
+    if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
+        return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
+    }
+    return { allowed: true };
+}
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *

package/internals/validation.d.ts

@@ -0,0 +1,9 @@
+import type { Dictionary } from '@crawlee/types';
+export { ArgumentValidationError, parseArgument } from '@apify/validations';
+/**
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
+ * into a second, half-working way of configuring the same pool.
+ * @internal
+ */
+export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;

package/internals/validation.js

@@ -0,0 +1,16 @@
+export { ArgumentValidationError, parseArgument } from '@apify/validations';
+/**
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
+ * into a second, half-working way of configuring the same pool.
+ * @internal
+ */
+export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
+    const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
+    if (names.length === 0) {
+        return;
+    }
+    throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
+        `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
+        'itself. Configure the pool you pass in instead.');
+}

package/package.json

@@ -1,13 +1,14 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -41,18 +42,19 @@
         "copy": "tsx ../../scripts/copy.ts"
     },
     "dependencies": {
-        "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
+        "@apify/validations": "^1.1.0",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
-        "domhandler": "^5.0.3",
+        "domhandler": "^6.0.0",
         "file-type": "^21.0.0",
-        "ow": "^2.0.0",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
+        "tldts": "^7.0.6",
         "tslib": "^2.8.1",
-        "whatwg-mimetype": "^4.0.0"
+        "whatwg-mimetype": "^5.0.0",
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -61,5 +63,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }