@crawlee/core 4.0.0-beta.121 → 4.0.0-beta.123

package/crawlers/crawler_commons.d.ts

@@ -1,12 +1,12 @@
 import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
+import type { ReadonlyDeep } from 'type-fest';
+import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
 import type { Request, RequestOptions, Source } from '../request.js';
 import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
 import type { KeyValueStore } from '../storages/key_value_store.js';
-import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 /**
@@ -37,7 +37,7 @@ export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
  * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
  * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
  */
-export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
 /**
  * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
  * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
@@ -87,36 +87,15 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      */
     pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
+     * Add requests directly to the request queue currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       include: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     */
-    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
-    /**
-     * Add requests directly to the request queue.
+     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
+     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
      *
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
     /**
      * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
      */
@@ -131,31 +110,6 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     log: CrawleeLogger;
 }
 export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       include: [
-     *           'https://www.example.com/handbags/*',
-     *       ],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
-     */
-    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
     /**
      * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *

package/crawlers/statistics.d.ts

@@ -1,5 +1,5 @@
 import type { CrawleeLogger } from '../log.js';
-import type { KeyValueStore } from '../storages/key_value_store.js';
+import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * Persistence-related options to control how and when crawler's data gets persisted.

package/crawlers/statistics.js

@@ -1,7 +1,8 @@
-import ow from 'ow';
 import { z } from 'zod';
 import { RecoverableState } from '../recoverable_state.js';
 import { serviceLocator } from '../service_locator.js';
+import { KeyValueStore } from '../storages/key_value_store.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * @ignore
@@ -17,6 +18,16 @@ class Job {
         return this.#durationMillis;
     }
 }
+const statisticsOptionsSchema = z.strictObject({
+    logIntervalSecs: schemas.anyNumber.default(60),
+    logMessage: z.string().default('Statistics'),
+    log: validators.logger.optional(),
+    keyValueStore: z.instanceof(KeyValueStore).optional(),
+    // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain copy).
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    saveErrorSnapshots: z.boolean().default(false),
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+});
 const errorTrackerConfig = {
     showErrorCode: true,
     showErrorName: true,
@@ -184,21 +195,10 @@ export class Statistics {
      * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            logIntervalSecs: ow.optional.number,
-            logMessage: ow.optional.string,
-            log: ow.optional.object,
-            keyValueStore: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-            saveErrorSnapshots: ow.optional.boolean,
-            id: ow.optional.any(ow.number, ow.string),
-        }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
-            enable: true,
-        }, saveErrorSnapshots = false, id, } = options;
+        const { logIntervalSecs, logMessage, log, keyValueStore, persistenceOptions, saveErrorSnapshots, id } = parseArgument(options, statisticsOptionsSchema);
         this.id = id ?? String(Statistics.id++);
         this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
-        this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
+        this.log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.#logIntervalMillis = logIntervalSecs * 1000;

package/debug.js

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import ow from 'ow';
+import { parseArgument, schemas } from './validators.js';
 /**
  * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
  *
@@ -12,9 +12,9 @@ import ow from 'ow';
  * @internal
  */
 export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
-    ow(request, ow.object);
-    ow(response, ow.object);
-    ow(additionalFields, ow.object);
+    parseArgument(request, schemas.anyObject);
+    parseArgument(response, schemas.anyObject);
+    parseArgument(additionalFields, schemas.anyObject);
     return {
         requestId: request.id,
         url: request.url,

package/enqueue_links/enqueue_links.d.ts

@@ -1,19 +1,24 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import { Request } from '../request.js';
-import type { IRequestManager } from '../storages/request_manager.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationOptions } from '../storages/request_queue.js';
-import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
+import type { Dictionary } from '@crawlee/types';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
+/**
+ * Options shared by the `extractLinks()` context helper across crawler types.
+ */
+export interface ExtractLinksOptions {
+    /** A CSS selector matching links to be extracted. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+}
+/**
+ * Options accepted by the `enqueueUrls()` context helper exposed by `BasicCrawler`.
+ */
+export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
-    /** An array of URLs to enqueue. */
-    urls?: readonly string[];
-    /** A request manager to which the URLs will be enqueued. */
-    requestManager?: IRequestManager;
-    /** A CSS selector matching links to be enqueued. */
-    selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
     /**
@@ -30,8 +35,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     skipNavigation?: boolean;
     /**
-     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
-     * since the relative URL resolution is done inside the browser automatically.
+     * A base URL that will be used to resolve relative URLs.
      */
     baseUrl?: string;
     /**
@@ -42,11 +46,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * Glob matching is always case-insensitive.
      * If you need case-sensitive matching, use a `RegExp`.
      *
-     * The patterns are combined with the {@link EnqueueLinksOptions.strategy|`strategy`} using AND logic - a URL
+     * The patterns are combined with the {@link EnqueueUrlsOptions.strategy|`strategy`} using AND logic - a URL
      * must match at least one `include` pattern **and** satisfy the strategy to be enqueued. To match URLs across
      * hostnames, pass an explicit {@link EnqueueStrategy.All} strategy.
      *
-     * If `undefined`, the links are enqueued based on the {@link EnqueueLinksOptions.strategy|`strategy`} alone.
+     * If `undefined`, the links are enqueued based on the {@link EnqueueUrlsOptions.strategy|`strategy`} alone.
      * Passing an empty array is not allowed.
      */
     include?: readonly UrlPatternInput[];
@@ -106,25 +110,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      *
      * @default EnqueueStrategy.SameHostname
      */
-    strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+    strategy?: EnqueueStrategyOption;
     /**
      * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
      * You can use this option to wait for adding all of them.
      */
     waitForAllRequestsToBeAdded?: boolean;
-    /**
-     * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
-     * If provided, disallowed URLs will be ignored.
-     */
-    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
-    /**
-     * Mirrors {@link BasicCrawlerOptions.respectRobotsTxtFile}: pass `false` to disable filtering or
-     * `{ userAgent }` to evaluate rules for a specific user-agent. Defaults to `*` when
-     * {@link EnqueueLinksOptions.robotsTxtFile|`robotsTxtFile`} is provided.
-     */
-    respectRobotsTxtFile?: boolean | {
-        userAgent?: string;
-    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
      * This is currently fired for requests skipped
@@ -134,6 +125,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
+/** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
+export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
 /**
  * The different enqueueing strategies available.
  *
@@ -185,34 +178,8 @@ export declare enum EnqueueStrategy {
      */
     SameOrigin = "same-origin"
 }
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   include: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
-    requestManager: {
-        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
-    };
-}): Promise<BatchAddRequestsResult>;
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -227,7 +194,12 @@ export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy
  */
 export interface ResolveBaseUrl {
     userProvidedBaseUrl?: string;
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];

package/enqueue_links/enqueue_links.js

@@ -1,7 +1,4 @@
-import ow from 'ow';
 import { getDomain } from 'tldts';
-import { Request } from '../request.js';
-import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, } from './shared.js';
 /**
  * The different enqueueing strategies available.
  *
@@ -54,155 +51,6 @@ export var EnqueueStrategy;
      */
     EnqueueStrategy["SameOrigin"] = "same-origin";
 })(EnqueueStrategy || (EnqueueStrategy = {}));
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   include: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export async function enqueueLinks(options) {
-    if (!options || Object.keys(options).length === 0) {
-        throw new RangeError([
-            'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
-            'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
-        ].join('\n'));
-    }
-    const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
-    ow(options, ow.object.exactShape({
-        urls: ow.array.ofType(ow.string),
-        requestManager: ow.object.hasKeys('addRequestsBatched'),
-        robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
-        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
-        onSkippedRequest: ow.optional.function,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        sessionId: ow.optional.string,
-        limit: ow.optional.number,
-        selector: ow.optional.string,
-        baseUrl: ow.optional.string,
-        userData: ow.optional.object,
-        label: ow.optional.string,
-        include: ow.optional.array.minLength(1).ofType(urlPatternValidator),
-        exclude: ow.optional.array.ofType(urlPatternValidator),
-        transformRequestFunction: ow.optional.function,
-        strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
-        waitForAllRequestsToBeAdded: ow.optional.boolean,
-    }));
-    const { requestManager, limit, urls, include, exclude, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
-    const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
-    const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
-    // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
-    // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
-    options.strategy ??= EnqueueStrategy.SameHostname;
-    const enqueueStrategyPatterns = [];
-    if (options.baseUrl) {
-        const url = new URL(options.baseUrl);
-        switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
-                // We need to get the origin of the passed in domain in the event someone sets baseUrl
-                // to an url like https://example.com/deep/default/path and one of the found urls is an
-                // absolute relative path (/path/to/page)
-                enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                break;
-            case EnqueueStrategy.SameDomain: {
-                // Get the actual hostname from the base url
-                const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
-                if (baseUrlHostname) {
-                    // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
-                    url.hostname = baseUrlHostname;
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                else {
-                    // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
-                    // as SameDomainAndSubdomain
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                break;
-            }
-            case EnqueueStrategy.SameOrigin: {
-                // The same behavior as SameHostname, but respecting the protocol of the URL
-                enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
-                break;
-            }
-            case EnqueueStrategy.All:
-            default:
-                enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
-                break;
-        }
-    }
-    async function reportSkippedRequests(skippedRequests, reason) {
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({
-                    url: request.url,
-                    reason: request.skippedReason ?? reason,
-                });
-            }));
-        }
-    }
-    let requestOptions = createRequestOptions(urls, options);
-    if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
-        const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
-        const skippedRequests = [];
-        requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
-                return true;
-            }
-            skippedRequests.push(request);
-            return false;
-        });
-        await reportSkippedRequests(skippedRequests, 'robotsTxt');
-    }
-    async function createFilteredRequests() {
-        const skippedRequests = [];
-        // Step 1: Filter request options by exclude patterns, user include patterns, and strategy patterns.
-        let filteredOptions;
-        if (urlPatternObjects.length === 0) {
-            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-        }
-        else {
-            // Filter by user patterns first (with exclude)
-            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
-            // ...then filter by the enqueue links strategy (making this an AND check)
-            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], options.strategy, (url) => skippedRequests.push(url));
-        }
-        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
-        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
-        if (transformRequestFunction) {
-            const skippedByTransform = [];
-            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
-            await reportSkippedRequests(skippedByTransform, 'transform');
-        }
-        // Step 3: Create Request instances from the final request options
-        return filteredOptions.map((opts) => new Request(opts));
-    }
-    const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
-        forefront,
-        waitForAllRequestsToBeAdded,
-        maxNewRequests: limit,
-    });
-    if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
-        await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
-    }
-    return { processedRequests: addedRequests, unprocessedRequests: [] };
-}
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -237,6 +85,41 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
     // before actually finding the urls
     return originalUrlOrigin;
 }
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
 /**
  * Internal function that changes the enqueue glob patterns to match both http and https
  */

package/enqueue_links/shared.d.ts

@@ -1,6 +1,7 @@
-import type { Awaitable } from '@crawlee/types';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import { z } from 'zod';
 import type { RequestOptions } from '../request.js';
-import type { EnqueueLinksOptions } from './enqueue_links.js';
+import type { EnqueueStrategyOption } from './enqueue_links.js';
 export { tryAbsoluteURL } from '@crawlee/utils/internal';
 export interface UrlPatternObject {
     glob?: string;
@@ -16,6 +17,11 @@ export interface RegExpObject {
 export type RegExpInput = RegExp | RegExpObject;
 /** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
 export type UrlPatternInput = GlobInput | RegExpInput;
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
 export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
 export type SkippedRequestCallback = (args: {
     url: string;
@@ -53,11 +59,18 @@ export declare function constructUrlPatternObjects(patterns: readonly UrlPattern
  * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedUrl?: (url: string) => void): RequestOptions[];
 /**
  * @ignore
  */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'sessionId' | 'strategy'>): RequestOptions[];
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
+    label?: string;
+    userData?: Dictionary;
+    baseUrl?: string;
+    skipNavigation?: boolean;
+    sessionId?: string;
+    strategy?: EnqueueStrategyOption;
+}): RequestOptions[];
 /**
  * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
  * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.

package/enqueue_links/shared.js

@@ -1,5 +1,7 @@
 import { URL } from 'node:url';
 import { Minimatch } from 'minimatch';
+import { z } from 'zod';
+import { schemas } from '../validators.js';
 export { tryAbsoluteURL } from '@crawlee/utils/internal';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
@@ -8,6 +10,16 @@ const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
  * @ignore
  */
 const enqueueLinksPatternCache = new Map();
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
 /**
  * @ignore
  */
@@ -132,6 +144,16 @@ export function filterRequestOptionsByPatterns(requestOptions, includePatterns,
     })
         .filter((opts) => opts !== null);
 }
+function isAbsoluteUrl(url) {
+    try {
+        // eslint-disable-next-line no-new
+        new URL(url);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * @ignore
  */
@@ -149,7 +171,12 @@ export function createRequestOptions(sources, options = {}) {
         }
     })
         .map((requestOptions) => {
-        requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
+        // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
+        // resolving against `baseUrl` in the first place.
+        if (!isAbsoluteUrl(requestOptions.url)) {
+            requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        }
         requestOptions.userData ??= options.userData ?? {};
         if (typeof options.label === 'string') {
             requestOptions.userData = {

package/memory-storage/resource-clients/dataset.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { BaseClient } from './common/base-client.js';
 /**
  * This is what API returns in the x-apify-pagination-limit
@@ -49,13 +49,7 @@ export class DatasetBackend extends BaseClient {
         this.updateTimestamps(true);
     }
     getData(options = {}) {
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
         return this.getDataPage({
             desc,
             offset: offset ?? 0,