@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/debug.js

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import ow from 'ow';
+import { parseArgument, schemas } from './validators.js';
 /**
  * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
  *
@@ -12,9 +12,9 @@ import ow from 'ow';
  * @internal
  */
 export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
-    ow(request, ow.object);
-    ow(response, ow.object);
-    ow(additionalFields, ow.object);
+    parseArgument(request, schemas.anyObject);
+    parseArgument(response, schemas.anyObject);
+    parseArgument(additionalFields, schemas.anyObject);
     return {
         requestId: request.id,
         url: request.url,

package/enqueue_links/enqueue_links.d.ts

@@ -1,26 +1,31 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import { Request } from '../request.js';
-import type { IRequestManager } from '../storages/request_manager.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationOptions } from '../storages/request_queue.js';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
+import type { Dictionary } from '@crawlee/types';
+import { EnqueueStrategy } from '@crawlee/utils';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
+/**
+ * Options shared by the `extractLinks()` context helper across crawler types.
+ */
+export interface ExtractLinksOptions {
+    /** A CSS selector matching links to be extracted. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+}
+/**
+ * Options accepted by the `enqueueUrls()` context helper exposed by `BasicCrawler`.
+ */
+export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
-    /** An array of URLs to enqueue. */
-    urls?: readonly string[];
-    /** A request manager to which the URLs will be enqueued. */
-    requestManager?: IRequestManager;
-    /** A CSS selector matching links to be enqueued. */
-    selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
     /**
      * Sets {@link Request.label} for newly enqueued requests.
      *
-     * This option has the lowest priority and can be overwritten by request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
+     * Can be overwritten by `transformRequestFunction`.
      */
     label?: string;
     /** Sets {@link Request.sessionId} for newly enqueued requests. */
@@ -31,66 +36,36 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     skipNavigation?: boolean;
     /**
-     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
-     * since the relative URL resolution is done inside the browser automatically.
+     * A base URL that will be used to resolve relative URLs.
      */
     baseUrl?: string;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
-     * enqueues the links with the same subdomain.
-     */
-    globs?: readonly GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * The patterns are combined with the {@link EnqueueUrlsOptions.strategy|`strategy`} using AND logic - a URL
+     * must match at least one `include` pattern **and** satisfy the strategy to be enqueued. To match URLs across
+     * hostnames, pass an explicit {@link EnqueueStrategy.All} strategy.
      *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
-     * enqueues the links with the same subdomain.
+     * If `undefined`, the links are enqueued based on the {@link EnqueueUrlsOptions.strategy|`strategy`} alone.
+     * Passing an empty array is not allowed.
      */
-    regexps?: readonly RegExpInput[];
+    include?: readonly UrlPatternInput[];
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues the links with the same subdomain.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: readonly PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * After request options are filtered by patterns, this function can be used
+     * After request options are filtered by `include`/`exclude` patterns, this function can be used
      * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
@@ -109,8 +84,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
      *
      * The function receives a {@link RequestOptions} object and can return either:
      * - The modified {@link RequestOptions} object
@@ -136,25 +111,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      *
      * @default EnqueueStrategy.SameHostname
      */
-    strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+    strategy?: EnqueueStrategyOption;
     /**
      * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
      * You can use this option to wait for adding all of them.
      */
     waitForAllRequestsToBeAdded?: boolean;
-    /**
-     * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
-     * If provided, disallowed URLs will be ignored.
-     */
-    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
-    /**
-     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
-     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
-     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
-     */
-    respectRobotsTxtFile?: boolean | {
-        userAgent?: string;
-    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
      * This is currently fired for requests skipped
@@ -164,86 +126,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   globs: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
-    requestManager: {
-        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
-    };
-}): Promise<BatchAddRequestsResult>;
+/** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
+export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
+export { EnqueueStrategy };
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -258,7 +145,12 @@ export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy
  */
 export interface ResolveBaseUrl {
     userProvidedBaseUrl?: string;
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];

package/enqueue_links/enqueue_links.js

@@ -1,234 +1,6 @@
-import ow from 'ow';
+import { EnqueueStrategy } from '@crawlee/utils';
 import { getDomain } from 'tldts';
-import { Request } from '../request.js';
-import { serviceLocator } from '../service_locator.js';
-import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, } from './shared.js';
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (EnqueueStrategy = {}));
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   globs: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export async function enqueueLinks(options) {
-    if (!options || Object.keys(options).length === 0) {
-        throw new RangeError([
-            'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
-            'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
-        ].join('\n'));
-    }
-    ow(options, ow.object.exactShape({
-        urls: ow.array.ofType(ow.string),
-        requestManager: ow.object.hasKeys('addRequestsBatched'),
-        robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
-        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
-        onSkippedRequest: ow.optional.function,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        sessionId: ow.optional.string,
-        limit: ow.optional.number,
-        selector: ow.optional.string,
-        baseUrl: ow.optional.string,
-        userData: ow.optional.object,
-        label: ow.optional.string,
-        pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-        globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-        regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-        transformRequestFunction: ow.optional.function,
-        strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
-        waitForAllRequestsToBeAdded: ow.optional.boolean,
-    }));
-    const { requestManager, limit, urls, 
-    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
-    pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
-    const urlExcludePatternObjects = [];
-    const urlPatternObjects = [];
-    if (exclude?.length) {
-        for (const excl of exclude) {
-            if (typeof excl === 'string' || 'glob' in excl) {
-                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-            }
-            else if (excl instanceof RegExp || 'regexp' in excl) {
-                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-            }
-        }
-    }
-    if (pseudoUrls?.length) {
-        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
-    if (!urlPatternObjects.length) {
-        options.strategy ??= EnqueueStrategy.SameHostname;
-    }
-    const enqueueStrategyPatterns = [];
-    if (options.baseUrl) {
-        const url = new URL(options.baseUrl);
-        switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
-                // We need to get the origin of the passed in domain in the event someone sets baseUrl
-                // to an url like https://example.com/deep/default/path and one of the found urls is an
-                // absolute relative path (/path/to/page)
-                enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                break;
-            case EnqueueStrategy.SameDomain: {
-                // Get the actual hostname from the base url
-                const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
-                if (baseUrlHostname) {
-                    // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
-                    url.hostname = baseUrlHostname;
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                else {
-                    // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
-                    // as SameDomainAndSubdomain
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                break;
-            }
-            case EnqueueStrategy.SameOrigin: {
-                // The same behavior as SameHostname, but respecting the protocol of the URL
-                enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
-                break;
-            }
-            case EnqueueStrategy.All:
-            default:
-                enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
-                break;
-        }
-    }
-    async function reportSkippedRequests(skippedRequests, reason) {
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({
-                    url: request.url,
-                    reason: request.skippedReason ?? reason,
-                });
-            }));
-        }
-    }
-    let requestOptions = createRequestOptions(urls, options);
-    if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
-        const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
-        const skippedRequests = [];
-        requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
-                return true;
-            }
-            skippedRequests.push(request);
-            return false;
-        });
-        await reportSkippedRequests(skippedRequests, 'robotsTxt');
-    }
-    async function createFilteredRequests() {
-        const skippedRequests = [];
-        // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
-        // Pattern-level options (label, userData, method, etc.) are merged during this step.
-        let filteredOptions;
-        if (urlPatternObjects.length === 0) {
-            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-        }
-        else {
-            // Filter by user patterns first (with exclude)
-            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-            // ...then filter by the enqueue links strategy (making this an AND check)
-            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], options.strategy, (url) => skippedRequests.push(url));
-        }
-        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
-        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
-        if (transformRequestFunction) {
-            const skippedByTransform = [];
-            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
-            await reportSkippedRequests(skippedByTransform, 'transform');
-        }
-        // Step 3: Create Request instances from the final request options
-        return filteredOptions.map((opts) => new Request(opts));
-    }
-    const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
-        forefront,
-        waitForAllRequestsToBeAdded,
-        maxNewRequests: limit,
-    });
-    if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
-        await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
-    }
-    return { processedRequests: addedRequests, unprocessedRequests: [] };
-}
+export { EnqueueStrategy };
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -264,7 +36,42 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
     return originalUrlOrigin;
 }
 /**
- * Internal function that changes the enqueue globs to match both http and https
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
+/**
+ * Internal function that changes the enqueue glob patterns to match both http and https
  */
 function ignoreHttpSchema(pattern) {
     return pattern.replace(/^(https?):\/\//, 'http{s,}://');