@crawlee/core 3.17.1-beta.6 → 3.17.1-beta.60

package/autoscaling/autoscaled_pool.js

@@ -415,14 +415,17 @@ class AutoscaledPool {
         this.isStopped = true;
         await new Promise((resolve, reject) => {
             let timeout;
+            let interval;
             if (timeoutSecs) {
                 timeout = setTimeout(() => {
+                    // Clean up the polling interval to prevent it from leaking on timeout.
+                    clearInterval(interval);
                     const err = new Error("The pool's running tasks did not finish" +
                         `in ${timeoutSecs} secs after pool.pause() invocation.`);
                     reject(err);
                 }, timeoutSecs);
             }
-            const interval = setInterval(() => {
+            interval = setInterval(() => {
                 if (this._currentConcurrency <= 0) {
                     // Clean up timeout and interval to prevent process hanging.
                     if (timeout)

package/autoscaling/snapshotter.d.ts

@@ -35,13 +35,13 @@ export interface SnapshotterOptions {
     /**
      * Defines the maximum number of new rate limit errors within
      * the given interval.
-     * @default 1
+     * @default 3
      */
     maxClientErrors?: number;
     /**
      * Sets the interval in seconds for which a history of resource snapshots
      * will be kept. Increasing this to very high numbers will affect performance.
-     * @default 60
+     * @default 30
      */
     snapshotHistorySecs?: number;
     /** @internal */

package/cookie_utils.js

@@ -107,6 +107,8 @@ function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = tough_cookie_1.Cookie.parse(cookieString);
+            if (!cookie)
+                throw new errors_1.CookieParseError(cookieString);
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });

package/enqueue_links/enqueue_links.d.ts

@@ -1,9 +1,10 @@
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
+import { EnqueueStrategy, type RobotsTxtFile } from '@crawlee/utils';
 import type { SetRequired } from 'type-fest';
 import type { Request } from '../request';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages';
 import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared';
+export { EnqueueStrategy };
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
@@ -156,57 +157,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.

package/enqueue_links/enqueue_links.js

@@ -4,62 +4,12 @@ exports.EnqueueStrategy = void 0;
 exports.enqueueLinks = enqueueLinks;
 exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
 const tslib_1 = require("tslib");
+const utils_1 = require("@crawlee/utils");
+Object.defineProperty(exports, "EnqueueStrategy", { enumerable: true, get: function () { return utils_1.EnqueueStrategy; } });
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const tldts_1 = require("tldts");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 const shared_1 = require("./shared");
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-var EnqueueStrategy;
-(function (EnqueueStrategy) {
-    /**
-     * Matches any URLs found
-     */
-    EnqueueStrategy["All"] = "all";
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameHostname"] = "same-hostname";
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    EnqueueStrategy["SameDomain"] = "same-domain";
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    EnqueueStrategy["SameOrigin"] = "same-origin";
-})(EnqueueStrategy || (exports.EnqueueStrategy = EnqueueStrategy = {}));
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
@@ -109,7 +59,7 @@ async function enqueueLinks(options) {
         exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
         regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
         transformRequestFunction: ow_1.default.optional.function,
-        strategy: ow_1.default.optional.string.oneOf(Object.values(EnqueueStrategy)),
+        strategy: ow_1.default.optional.string.oneOf(Object.values(utils_1.EnqueueStrategy)),
         waitForAllRequestsToBeAdded: ow_1.default.optional.boolean,
     }));
     const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
@@ -136,19 +86,19 @@ async function enqueueLinks(options) {
         urlPatternObjects.push(...(0, shared_1.constructRegExpObjectsFromRegExps)(regexps));
     }
     if (!urlPatternObjects.length) {
-        options.strategy ?? (options.strategy = EnqueueStrategy.SameHostname);
+        options.strategy ?? (options.strategy = utils_1.EnqueueStrategy.SameHostname);
     }
     const enqueueStrategyPatterns = [];
     if (options.baseUrl) {
         const url = new URL(options.baseUrl);
         switch (options.strategy) {
-            case EnqueueStrategy.SameHostname:
+            case utils_1.EnqueueStrategy.SameHostname:
                 // We need to get the origin of the passed in domain in the event someone sets baseUrl
                 // to an url like https://example.com/deep/default/path and one of the found urls is an
                 // absolute relative path (/path/to/page)
                 enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
                 break;
-            case EnqueueStrategy.SameDomain: {
+            case utils_1.EnqueueStrategy.SameDomain: {
                 // Get the actual hostname from the base url
                 const baseUrlHostname = (0, tldts_1.getDomain)(url.hostname, { mixedInputs: false });
                 if (baseUrlHostname) {
@@ -163,12 +113,12 @@ async function enqueueLinks(options) {
                 }
                 break;
             }
-            case EnqueueStrategy.SameOrigin: {
+            case utils_1.EnqueueStrategy.SameOrigin: {
                 // The same behavior as SameHostname, but respecting the protocol of the URL
                 enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
                 break;
             }
-            case EnqueueStrategy.All:
+            case utils_1.EnqueueStrategy.All:
             default:
                 enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
                 break;
@@ -249,13 +199,13 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
     const originalUrlOrigin = new URL(originalRequestUrl).origin;
     const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
     // We can assume users want to go off the domain in this case
-    if (enqueueStrategy === EnqueueStrategy.All) {
+    if (enqueueStrategy === utils_1.EnqueueStrategy.All) {
         return finalUrlOrigin;
     }
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
     // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
     // which is the intended behavior (since we went off domain)
-    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
+    if (enqueueStrategy === utils_1.EnqueueStrategy.SameDomain) {
         const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
         const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
         if (originalHostname === finalHostname) {

package/errors.d.ts

@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
  */
 export declare class MissingRouteError extends CriticalError {
 }
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+export declare class RequestValidationError extends NonRetryableError {
+    readonly label: string | symbol;
+    readonly issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[];
+    constructor(label: string | symbol, issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[]);
+}
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SessionError = exports.RetryRequestError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
+exports.SessionError = exports.RetryRequestError = exports.RequestValidationError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
 /**
  * Errors of `NonRetryableError` type will never be retried by the crawler.
  */
@@ -20,6 +20,37 @@ exports.CriticalError = CriticalError;
 class MissingRouteError extends CriticalError {
 }
 exports.MissingRouteError = MissingRouteError;
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+class RequestValidationError extends NonRetryableError {
+    constructor(label, issues) {
+        const details = issues
+            .map((issue) => {
+            const path = (issue.path ?? [])
+                .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+                .join('.');
+            return `- ${path ? `${path}: ` : ''}${issue.message}`;
+        })
+            .join('\n');
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
+        Object.defineProperty(this, "label", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: label
+        });
+        Object.defineProperty(this, "issues", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: issues
+        });
+    }
+}
+exports.RequestValidationError = RequestValidationError;
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/http_clients/got-scraping-http-client.js

@@ -22,6 +22,14 @@ class GotScrapingHttpClient {
         });
         return {
             ...gotResult,
+            complete: gotResult.complete,
+            headers: gotResult.headers,
+            ip: gotResult.ip,
+            redirectUrls: gotResult.redirectUrls,
+            statusCode: gotResult.statusCode,
+            statusMessage: gotResult.statusMessage,
+            trailers: gotResult.trailers,
+            url: gotResult.url,
             body: gotResult.body,
             request: { url: request.url, ...gotResult.request },
         };

package/index.mjs

@@ -46,6 +46,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -73,6 +74,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -93,5 +95,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.6",
+    "version": "3.17.1-beta.60",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -57,12 +57,13 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
         "@apify/pseudo_url": "^2.0.30",
-        "@apify/timeout": "^0.3.0",
+        "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.6",
-        "@crawlee/types": "3.17.1-beta.6",
-        "@crawlee/utils": "3.17.1-beta.6",
+        "@crawlee/memory-storage": "3.17.1-beta.60",
+        "@crawlee/types": "3.17.1-beta.60",
+        "@crawlee/utils": "3.17.1-beta.60",
         "@sapphire/async-queue": "^1.5.1",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
@@ -83,5 +84,5 @@
             }
         }
     },
-    "gitHead": "eda1d0870cfa01cfd7d2a4a9d0e4e6b99d18dea1"
+    "gitHead": "5ba2517b5f3bc6ef868b860c6c783a3e7e7d4a93"
 }

package/router.d.ts

@@ -1,14 +1,57 @@
 import type { Dictionary } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons';
 import type { Request } from './request';
 import type { Awaitable } from './typedefs';
-export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext> extends Router<Context> {
+/**
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
+ * that falls through to the default handler (i.e. whose label has no route of its own).
+ */
+export declare const defaultRoute: unique symbol;
+/**
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ */
+export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+    request: LoadedRequest<Request<UserData>>;
+};
+/**
+ * A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
+ * validating that label's `request.userData`. Pass it to {@link Router.create} or a `createXRouter`
+ * factory to derive the per-label `request.userData` types *and* validate them at runtime. The optional
+ * {@link defaultRoute} key registers a schema for requests handled by the default route.
+ */
+export type RouteSchemas = Record<string, StandardSchemaV1> & {
+    [defaultRoute]?: StandardSchemaV1;
+};
+/**
+ * Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each
+ * schema's output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
+ * {@link defaultRoute} schema drives runtime validation only, so it is excluded from the typed route map.
+ */
+export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
+    [Label in Extract<keyof Schemas, string>]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary ? StandardSchemaV1.InferOutput<Schemas[Label]> : Dictionary;
+};
+/**
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
+ * value. Throws a {@link RequestValidationError} when validation fails.
+ * @internal
+ */
+export declare function validateUserData(label: string | symbol, schema: StandardSchemaV1, userData: unknown): Promise<Dictionary>;
+/**
+ * The set of labels accepted by {@link Router.addHandler}. When the router declares a concrete
+ * route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
+ * allowed — unknown labels become a compile-time error. When the map is left open (the default
+ * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
+ */
+export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
+export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
 export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
-export type RouterRoutes<Context, UserData extends Dictionary> = {
-    [label in string | symbol]: (ctx: Omit<Context, 'request'> & {
-        request: Request<UserData>;
+export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
+    [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
+        request: Request<Routes[Label]>;
     }) => Awaitable<void>;
 };
 /**
@@ -75,9 +118,51 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
+ * throws a {@link RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
  */
-export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
+export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     private readonly routes;
+    private readonly schemas;
     private readonly middlewares;
     /**
      * use Router.create() instead!
@@ -85,17 +170,28 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      */
     protected constructor();
     /**
-     * Registers new route handler for given label.
+     * Registers new route handler for given label. When the router declares a route map, the
+     * `label` is restricted to the declared labels and `request.userData` is typed accordingly.
+     */
+    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>): void;
+    /**
+     * Registers new route handler for given label, explicitly typing `request.userData` via the
+     * `UserData` type argument. Useful when the router has no declared route map (the open default)
+     * and you want to type a single handler, or to register a handler under a `symbol` label.
      */
-    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx: Omit<Context, 'request'> & {
-        request: LoadedRequest<Request<UserData>>;
-    }) => Awaitable<void>): void;
+    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
     /**
-     * Registers default route handler.
+     * Registers default route handler. As a fallback it can receive any request (including labels not
+     * declared in the route map), so `request.userData` defaults to the context's `userData` type
+     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
-    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: Omit<Context, 'request'> & {
-        request: LoadedRequest<Request<UserData>>;
-    }) => Awaitable<void>): void;
+    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
+    /**
+     * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
+     * to validate `request.userData` when requests are added.
+     * @internal
+     */
+    getSchema(label?: string | symbol): StandardSchemaV1 | undefined;
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
@@ -105,6 +201,11 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
     getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@link RequestValidationError} when validation fails.
+     */
+    private validateRequest;
     /**
      * Throws when the label already exists in our registry.
      */
@@ -129,5 +230,7 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * await crawler.run();
      * ```
      */
-    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 }

package/router.js

@@ -1,8 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Router = void 0;
+exports.Router = exports.defaultRoute = void 0;
+exports.validateUserData = validateUserData;
 const errors_1 = require("./errors");
-const defaultRoute = Symbol('default-route');
+/**
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
+ * that falls through to the default handler (i.e. whose label has no route of its own).
+ */
+exports.defaultRoute = Symbol('default-route');
+/** Whether a validation issue points at the top-level `label` key. */
+function isLabelIssue(issue) {
+    if (issue.path?.length !== 1) {
+        return false;
+    }
+    const [segment] = issue.path;
+    return (typeof segment === 'object' ? segment.key : segment) === 'label';
+}
+/**
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
+ * value. Throws a {@link RequestValidationError} when validation fails.
+ * @internal
+ */
+async function validateUserData(label, schema, userData) {
+    const { label: _label, ...rest } = (userData ?? {});
+    // `label` is a Crawlee-managed key that lives inside `userData`, so validating it is opt-in: we validate
+    // without it first, letting schemas that don't describe it pass (including `.strict()` ones). A schema that
+    // *does* declare `label` reports an issue for the now-missing key — so we re-validate with it included,
+    // honouring the declaration. Unlike `userData.__crawlee`, `label` is enumerable, so schemas do see it.
+    let result = await schema['~standard'].validate(rest);
+    if (result.issues?.some(isLabelIssue)) {
+        result = await schema['~standard'].validate({ ...rest, label });
+    }
+    if (result.issues) {
+        throw new errors_1.RequestValidationError(label, result.issues);
+    }
+    // Restore the label so it survives schemas that strip undeclared keys.
+    return { ...result.value, label };
+}
 /**
  * Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
  *
@@ -67,6 +102,47 @@ const defaultRoute = Symbol('default-route');
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
+ * throws a {@link RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
  */
 class Router {
     /**
@@ -80,6 +156,12 @@ class Router {
             writable: true,
             value: new Map()
         });
+        Object.defineProperty(this, "schemas", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
         Object.defineProperty(this, "middlewares", {
             enumerable: true,
             configurable: true,
@@ -87,19 +169,38 @@ class Router {
             value: []
         });
     }
-    /**
-     * Registers new route handler for given label.
-     */
     addHandler(label, handler) {
         this.validate(label);
         this.routes.set(label, handler);
     }
     /**
-     * Registers default route handler.
+     * Registers default route handler. As a fallback it can receive any request (including labels not
+     * declared in the route map), so `request.userData` defaults to the context's `userData` type
+     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler(handler) {
-        this.validate(defaultRoute);
-        this.routes.set(defaultRoute, handler);
+        this.validate(exports.defaultRoute);
+        this.routes.set(exports.defaultRoute, handler);
+    }
+    /**
+     * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
+     * to validate `request.userData` when requests are added.
+     * @internal
+     */
+    getSchema(label) {
+        if (label != null) {
+            const schema = this.schemas.get(label);
+            if (schema) {
+                return schema;
+            }
+            // A label with its own route is fully specified; don't fall back to the default-route schema.
+            if (this.routes.has(label)) {
+                return undefined;
+            }
+        }
+        // Requests with no route of their own fall through to the default handler, so validate their
+        // `userData` against the default-route schema, if one was registered.
+        return this.schemas.get(exports.defaultRoute);
     }
     /**
      * Registers a middleware that will be fired before the matching route handler.
@@ -115,57 +216,57 @@ class Router {
         if (label && this.routes.has(label)) {
             return this.routes.get(label);
         }
-        if (this.routes.has(defaultRoute)) {
-            return this.routes.get(defaultRoute);
+        if (this.routes.has(exports.defaultRoute)) {
+            return this.routes.get(exports.defaultRoute);
         }
         throw new errors_1.MissingRouteError(`Route not found for label '${String(label)}'.` +
             ' You must set up a route for this label or a default route.' +
             ' Use `requestHandler`, `router.addHandler` or `router.addDefaultHandler`.');
     }
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@link RequestValidationError} when validation fails.
+     */
+    async validateRequest(context) {
+        const label = context.request.label;
+        const schema = this.getSchema(label);
+        if (schema) {
+            context.request.userData = (await validateUserData(label, schema, context.request.userData));
+        }
+    }
     /**
      * Throws when the label already exists in our registry.
      */
     validate(label) {
         if (this.routes.has(label)) {
-            const message = label === defaultRoute
+            const message = label === exports.defaultRoute
                 ? `Default route is already defined!`
                 : `Route for label '${String(label)}' is already defined!`;
             throw new Error(message);
         }
     }
-    /**
-     * Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
-     *
-     * ```ts
-     * import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
-     *
-     * const router = Router.create<CheerioCrawlingContext>();
-     * router.addHandler('label-a', async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     * router.addDefaultHandler(async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     *
-     * const crawler = new CheerioCrawler({
-     *     requestHandler: router,
-     * });
-     * await crawler.run();
-     * ```
-     */
-    static create(routes) {
+    static create(routesOrSchemas) {
         const router = new Router();
         const obj = Object.create(Function.prototype);
         obj.addHandler = router.addHandler.bind(router);
         obj.addDefaultHandler = router.addDefaultHandler.bind(router);
+        obj.getSchema = router.getSchema.bind(router);
         obj.getHandler = router.getHandler.bind(router);
         obj.use = router.use.bind(router);
-        for (const [label, handler] of Object.entries(routes ?? {})) {
-            router.addHandler(label, handler);
+        // `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
+        for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {
+            const value = routesOrSchemas[label];
+            if (typeof value === 'function') {
+                router.addHandler(label, value);
+            }
+            else {
+                router.schemas.set(label, value);
+            }
         }
         const func = async function (context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
+            await router.validateRequest(context);
             for (const middleware of router.middlewares) {
                 await middleware(context);
             }

package/session_pool/session_pool.d.ts

@@ -192,8 +192,11 @@ export declare class SessionPool extends EventEmitter {
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    teardown(): Promise<void>;
+    teardown({ persistState }?: {
+        persistState?: boolean;
+    }): Promise<void>;
     /**
      * SessionPool should not work before initialization.
      */

package/session_pool/session_pool.js

@@ -342,10 +342,13 @@ class SessionPool extends node_events_1.EventEmitter {
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    async teardown() {
+    async teardown({ persistState = true } = {}) {
         this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        await this.persistState();
+        if (persistState) {
+            await this.persistState();
+        }
     }
     /**
      * SessionPool should not work before initialization.

package/storages/request_dedup_cache.d.ts

@@ -0,0 +1,23 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export declare class RequestDeduplicationCache {
+    private readonly size;
+    private keys;
+    private ids;
+    constructor(size?: number);
+    get(cacheKey: string): string | null;
+    add(cacheKey: string, requestId: string): void;
+    clear(): void;
+    private indexOf;
+}

package/storages/request_dedup_cache.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestDeduplicationCache = void 0;
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+class RequestDeduplicationCache {
+    // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
+    constructor(size = 1000000) {
+        Object.defineProperty(this, "size", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: size
+        });
+        Object.defineProperty(this, "keys", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "ids", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.keys = new Array(size);
+        this.ids = new Array(size);
+    }
+    get(cacheKey) {
+        const index = this.indexOf(cacheKey);
+        return this.keys[index] === cacheKey ? this.ids[index] : null;
+    }
+    add(cacheKey, requestId) {
+        const index = this.indexOf(cacheKey);
+        this.keys[index] = cacheKey;
+        this.ids[index] = requestId;
+    }
+    clear() {
+        this.keys = new Array(this.size);
+        this.ids = new Array(this.size);
+    }
+    // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
+    indexOf(cacheKey) {
+        /* eslint-disable no-bitwise */
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < cacheKey.length; i++) {
+            hash ^= cacheKey.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193);
+        }
+        return (hash >>> 0) % this.size;
+        /* eslint-enable no-bitwise */
+    }
+}
+exports.RequestDeduplicationCache = RequestDeduplicationCache;

package/storages/request_list.d.ts

@@ -336,6 +336,12 @@ export declare class RequestList implements IRequestList {
      * @inheritDoc
      */
     persistState(): Promise<void>;
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    teardown(): Promise<void>;
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),

package/storages/request_list.js

@@ -248,6 +248,7 @@ class RequestList {
         this.sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
         this.proxyConfiguration = proxyConfiguration;
+        this.persistState = this.persistState.bind(this);
     }
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
@@ -273,7 +274,7 @@ class RequestList {
         if (this.persistRequestsKey && !this.areRequestsPersisted)
             await this._persistRequests();
         if (this.persistStateKey) {
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState.bind(this));
+            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
         }
         return this;
     }
@@ -324,9 +325,11 @@ class RequestList {
                 const sourcesFromFunction = await this.sourcesFunction();
                 const sourcesFromFunctionCount = sourcesFromFunction.length;
                 for (let i = 0; i < sourcesFromFunctionCount; i++) {
-                    const source = sourcesFromFunction.shift();
+                    const source = sourcesFromFunction[i];
+                    delete sourcesFromFunction[i];
                     this._addRequest(source);
                 }
+                sourcesFromFunction.length = 0;
             }
             catch (e) {
                 const err = e;
@@ -353,6 +356,17 @@ class RequestList {
             this.log.exception(err, 'Attempted to persist state, but failed.');
         }
     }
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    async teardown() {
+        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        if (this.persistStateKey) {
+            await this.persistState();
+        }
+    }
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),

package/storages/request_provider.d.ts

@@ -5,6 +5,7 @@ import { Configuration } from '../configuration';
 import type { ProxyConfiguration } from '../proxy_configuration';
 import type { InternalSource, RequestOptions, Source } from '../request';
 import { Request } from '../request';
+import { RequestDeduplicationCache } from './request_dedup_cache';
 import type { IStorage, StorageManagerOptions } from './storage_manager';
 export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
 /**
@@ -74,6 +75,12 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
     private initialHandledCount;
     protected queueHeadIds: ListDictionary<string>;
     protected requestCache: LruCache<RequestLruItem>;
+    /**
+     * Remembers the `requestId` of every request already submitted to the client — including background
+     * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
+     * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
+     */
+    protected requestSeenCache: RequestDeduplicationCache;
     protected recentlyHandledRequestsCache: LruCache<boolean>;
     protected queuePausedForMigration: boolean;
     protected lastActivity: Date;

package/storages/request_provider.js

@@ -11,8 +11,14 @@ const configuration_1 = require("../configuration");
 const log_1 = require("../log");
 const request_1 = require("../request");
 const access_checking_1 = require("./access_checking");
+const request_dedup_cache_1 = require("./request_dedup_cache");
 const storage_manager_1 = require("./storage_manager");
 const utils_2 = require("./utils");
+/**
+ * Maximum number of consecutive batch-add attempts that make no progress before the remaining
+ * unprocessed requests are skipped, so permanently rejected requests don't retry forever.
+ */
+const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
 class RequestProvider {
     constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
         Object.defineProperty(this, "config", {
@@ -113,6 +119,17 @@ class RequestProvider {
             writable: true,
             value: void 0
         });
+        /**
+         * Remembers the `requestId` of every request already submitted to the client — including background
+         * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
+         * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
+         */
+        Object.defineProperty(this, "requestSeenCache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "recentlyHandledRequestsCache", {
             enumerable: true,
             configurable: true,
@@ -151,6 +168,7 @@ class RequestProvider {
         });
         this.proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new datastructures_1.LruCache({ maxLength: options.requestCacheMaxSize });
+        this.requestSeenCache = new request_dedup_cache_1.RequestDeduplicationCache();
         this.recentlyHandledRequestsCache = new datastructures_1.LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
         this.log = log_1.log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
         const eventManager = config.getEventManager();
@@ -227,6 +245,7 @@ class RequestProvider {
         };
         const { requestId, wasAlreadyPresent } = queueOperationInfo;
         this._cacheRequest(cacheKey, queueOperationInfo);
+        this.requestSeenCache.add(cacheKey, requestId);
         if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
             this.assumedTotalCount++;
             // Performance optimization: add request straight to head if possible
@@ -288,16 +307,17 @@ class RequestProvider {
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
+            // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
             const cachedInfo = this.requestCache.get(cacheKey);
-            if (cachedInfo) {
-                request.id = cachedInfo.id;
+            const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+            if (knownRequestId) {
+                request.id = knownRequestId;
                 results.processedRequests.push({
                     wasAlreadyPresent: true,
-                    // We may assume that if request is in local cache then also the information if the
-                    // request was already handled is there because just one client should be using one queue.
-                    wasAlreadyHandled: cachedInfo.isHandled,
-                    requestId: cachedInfo.id,
-                    uniqueKey: cachedInfo.uniqueKey,
+                    // The dedup cache doesn't track the handled state; only the full record does.
+                    wasAlreadyHandled: cachedInfo?.isHandled ?? false,
+                    requestId: knownRequestId,
+                    uniqueKey: request.uniqueKey,
                 });
             }
             else if (!requestsToAdd.has(request.uniqueKey)) {
@@ -320,6 +340,8 @@ class RequestProvider {
             if (cache) {
                 this._cacheRequest(cacheKey, { ...newRequest, forefront });
             }
+            // Unlike `requestCache`, populate this on every batch (including background ones).
+            this.requestSeenCache.add(cacheKey, requestId);
             if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
                 this.assumedTotalCount++;
                 // Performance optimization: add request straight to head if possible
@@ -385,13 +407,20 @@ class RequestProvider {
         const requestIterator = generateRequests();
         const chunks = (0, utils_1.peekableAsyncIterable)((0, utils_1.chunkedAsyncIterable)(requestIterator, effectiveChunkSize));
         const chunksIterator = chunks[Symbol.asyncIterator]();
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
+        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true, unsuccessfulAttempts = 0) => {
             const resultsToReturn = [];
             const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
             resultsToReturn.push(...apiResult.processedRequests);
             if (apiResult.unprocessedRequests.length) {
+                // Count attempts that make no progress, so permanently rejected requests (e.g. a malformed
+                // `userData` shape causing a 400) don't loop forever. Any progress resets the counter.
+                const attempts = apiResult.processedRequests.length ? 0 : unsuccessfulAttempts + 1;
+                if (attempts >= MAX_UNPROCESSED_REQUESTS_RETRIES) {
+                    this.log.warning(`Some requests were consistently rejected by the request queue and will be skipped after ${MAX_UNPROCESSED_REQUESTS_RETRIES} attempts. This usually means the request data is malformed (e.g. an invalid 'userData' shape).`, { unprocessedRequests: apiResult.unprocessedRequests });
+                    return resultsToReturn;
+                }
                 await (0, utils_1.sleep)(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false)));
+                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false, attempts)));
             }
             return resultsToReturn;
         };
@@ -546,6 +575,7 @@ class RequestProvider {
         this.assumedTotalCount = 0;
         this.assumedHandledCount = 0;
         this.requestCache.clear();
+        this.requestSeenCache.clear();
     }
     /**
      * Caches information about request to beware of unneeded addRequest() calls.

package/storages/sitemap_request_list.d.ts

@@ -1,6 +1,7 @@
 import { type ParseSitemapOptions } from '@crawlee/utils';
 import { Configuration } from '../configuration';
 import type { GlobInput, RegExpInput } from '../enqueue_links';
+import { EnqueueStrategy } from '../enqueue_links';
 import { Request } from '../request';
 import type { IRequestList } from './request_list';
 interface UrlConstraints {
@@ -76,6 +77,13 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      * @default 200
      */
     maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Advanced options for the underlying `parseSitemap` call.
      */
@@ -133,6 +141,10 @@ export declare class SitemapRequestList implements IRequestList {
      * Proxy URL to be used for sitemap loading.
      */
     private proxyUrl;
+    /**
+     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+     */
+    private enqueueStrategy;
     /**
      * Logger instance.
      */

package/storages/sitemap_request_list.js

@@ -132,6 +132,15 @@ class SitemapRequestList {
             writable: true,
             value: void 0
         });
+        /**
+         * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+         */
+        Object.defineProperty(this, "enqueueStrategy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * Logger instance.
          */
@@ -173,6 +182,7 @@ class SitemapRequestList {
             signal: ow_1.default.optional.any(),
             timeoutMillis: ow_1.default.optional.number,
             maxBufferSize: ow_1.default.optional.number,
+            enqueueStrategy: ow_1.default.optional.string.oneOf(Object.values(enqueue_links_1.EnqueueStrategy)),
             parseSitemapOptions: ow_1.default.optional.object,
             globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
             exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
@@ -200,6 +210,7 @@ class SitemapRequestList {
         this.persistStateKey = options.persistStateKey;
         this.persistenceOptions = { enable: true, ...options.persistenceOptions };
         this.proxyUrl = options.proxyUrl;
+        this.enqueueStrategy = options.enqueueStrategy ?? enqueue_links_1.EnqueueStrategy.SameHostname;
         this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
         this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
         this.events = config.getEventManager();
@@ -308,6 +319,7 @@ class SitemapRequestList {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
+                    enqueueStrategy: this.enqueueStrategy,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
@@ -457,7 +469,7 @@ class SitemapRequestList {
             if (!nextUrl) {
                 return null;
             }
-            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl }));
+            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));
         }
         this.inProgress.add(nextUrl);
         return this.requestData.get(nextUrl);