@crawlee/core 3.17.1-beta.53 → 3.17.1-beta.55

package/errors.d.ts

@@ -14,6 +14,26 @@ export declare class CriticalError extends NonRetryableError {
  */
 export declare class MissingRouteError extends CriticalError {
 }
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+export declare class RequestValidationError extends NonRetryableError {
+    readonly label: string | symbol;
+    readonly issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[];
+    constructor(label: string | symbol, issues: readonly {
+        readonly message: string;
+        readonly path?: readonly (PropertyKey | {
+            key: PropertyKey;
+        })[];
+    }[]);
+}
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SessionError = exports.RetryRequestError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
+exports.SessionError = exports.RetryRequestError = exports.RequestValidationError = exports.MissingRouteError = exports.CriticalError = exports.NonRetryableError = void 0;
 /**
  * Errors of `NonRetryableError` type will never be retried by the crawler.
  */
@@ -20,6 +20,37 @@ exports.CriticalError = CriticalError;
 class MissingRouteError extends CriticalError {
 }
 exports.MissingRouteError = MissingRouteError;
+/**
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+class RequestValidationError extends NonRetryableError {
+    constructor(label, issues) {
+        const details = issues
+            .map((issue) => {
+            const path = (issue.path ?? [])
+                .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+                .join('.');
+            return `- ${path ? `${path}: ` : ''}${issue.message}`;
+        })
+            .join('\n');
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
+        Object.defineProperty(this, "label", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: label
+        });
+        Object.defineProperty(this, "issues", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: issues
+        });
+    }
+}
+exports.RequestValidationError = RequestValidationError;
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

package/index.mjs

@@ -46,6 +46,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -73,6 +74,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -93,5 +95,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.53",
+    "version": "3.17.1-beta.55",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,10 +59,11 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.53",
-        "@crawlee/types": "3.17.1-beta.53",
-        "@crawlee/utils": "3.17.1-beta.53",
+        "@crawlee/memory-storage": "3.17.1-beta.55",
+        "@crawlee/types": "3.17.1-beta.55",
+        "@crawlee/utils": "3.17.1-beta.55",
         "@sapphire/async-queue": "^1.5.1",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
@@ -83,5 +84,5 @@
             }
         }
     },
-    "gitHead": "59a0d19d96e7402cd8d0edd8b90ad4f5abc8f7db"
+    "gitHead": "93d29e305f01ca6ca9bb79264a5ca553b19766ef"
 }

package/router.d.ts

@@ -1,13 +1,43 @@
 import type { Dictionary } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons';
 import type { Request } from './request';
 import type { Awaitable } from './typedefs';
+/**
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
+ * that falls through to the default handler (i.e. whose label has no route of its own).
+ */
+export declare const defaultRoute: unique symbol;
 /**
  * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
  */
 export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
     request: LoadedRequest<Request<UserData>>;
 };
+/**
+ * A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
+ * validating that label's `request.userData`. Pass it to {@link Router.create} or a `createXRouter`
+ * factory to derive the per-label `request.userData` types *and* validate them at runtime. The optional
+ * {@link defaultRoute} key registers a schema for requests handled by the default route.
+ */
+export type RouteSchemas = Record<string, StandardSchemaV1> & {
+    [defaultRoute]?: StandardSchemaV1;
+};
+/**
+ * Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each
+ * schema's output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
+ * {@link defaultRoute} schema drives runtime validation only, so it is excluded from the typed route map.
+ */
+export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
+    [Label in Extract<keyof Schemas, string>]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary ? StandardSchemaV1.InferOutput<Schemas[Label]> : Dictionary;
+};
+/**
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
+ * value. Throws a {@link RequestValidationError} when validation fails.
+ * @internal
+ */
+export declare function validateUserData(label: string | symbol, schema: StandardSchemaV1, userData: unknown): Promise<Dictionary>;
 /**
  * The set of labels accepted by {@link Router.addHandler}. When the router declares a concrete
  * route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
@@ -110,9 +140,29 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
  *
  * router.addHandler('TYPO', async () => {}); // compile error: not a known label
  * ```
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
+ * throws a {@link RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
  */
 export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     private readonly routes;
+    private readonly schemas;
     private readonly middlewares;
     /**
      * use Router.create() instead!
@@ -136,6 +186,12 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
+    /**
+     * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
+     * to validate `request.userData` when requests are added.
+     * @internal
+     */
+    getSchema(label?: string | symbol): StandardSchemaV1 | undefined;
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
@@ -145,6 +201,11 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
     getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@link RequestValidationError} when validation fails.
+     */
+    private validateRequest;
     /**
      * Throws when the label already exists in our registry.
      */
@@ -171,4 +232,5 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      */
     static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
     static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 }

package/router.js

@@ -1,8 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Router = void 0;
+exports.Router = exports.defaultRoute = void 0;
+exports.validateUserData = validateUserData;
 const errors_1 = require("./errors");
-const defaultRoute = Symbol('default-route');
+/**
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
+ * that falls through to the default handler (i.e. whose label has no route of its own).
+ */
+exports.defaultRoute = Symbol('default-route');
+/** Whether a validation issue points at the top-level `label` key. */
+function isLabelIssue(issue) {
+    if (issue.path?.length !== 1) {
+        return false;
+    }
+    const [segment] = issue.path;
+    return (typeof segment === 'object' ? segment.key : segment) === 'label';
+}
+/**
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
+ * value. Throws a {@link RequestValidationError} when validation fails.
+ * @internal
+ */
+async function validateUserData(label, schema, userData) {
+    const { label: _label, ...rest } = (userData ?? {});
+    // `label` is a Crawlee-managed key that lives inside `userData`, so validating it is opt-in: we validate
+    // without it first, letting schemas that don't describe it pass (including `.strict()` ones). A schema that
+    // *does* declare `label` reports an issue for the now-missing key — so we re-validate with it included,
+    // honouring the declaration. Unlike `userData.__crawlee`, `label` is enumerable, so schemas do see it.
+    let result = await schema['~standard'].validate(rest);
+    if (result.issues?.some(isLabelIssue)) {
+        result = await schema['~standard'].validate({ ...rest, label });
+    }
+    if (result.issues) {
+        throw new errors_1.RequestValidationError(label, result.issues);
+    }
+    // Restore the label so it survives schemas that strip undeclared keys.
+    return { ...result.value, label };
+}
 /**
  * Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
  *
@@ -89,6 +124,25 @@ const defaultRoute = Symbol('default-route');
  *
  * router.addHandler('TYPO', async () => {}); // compile error: not a known label
  * ```
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
+ * throws a {@link RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
  */
 class Router {
     /**
@@ -102,6 +156,12 @@ class Router {
             writable: true,
             value: new Map()
         });
+        Object.defineProperty(this, "schemas", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
         Object.defineProperty(this, "middlewares", {
             enumerable: true,
             configurable: true,
@@ -119,8 +179,28 @@ class Router {
      * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler(handler) {
-        this.validate(defaultRoute);
-        this.routes.set(defaultRoute, handler);
+        this.validate(exports.defaultRoute);
+        this.routes.set(exports.defaultRoute, handler);
+    }
+    /**
+     * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
+     * to validate `request.userData` when requests are added.
+     * @internal
+     */
+    getSchema(label) {
+        if (label != null) {
+            const schema = this.schemas.get(label);
+            if (schema) {
+                return schema;
+            }
+            // A label with its own route is fully specified; don't fall back to the default-route schema.
+            if (this.routes.has(label)) {
+                return undefined;
+            }
+        }
+        // Requests with no route of their own fall through to the default handler, so validate their
+        // `userData` against the default-route schema, if one was registered.
+        return this.schemas.get(exports.defaultRoute);
     }
     /**
      * Registers a middleware that will be fired before the matching route handler.
@@ -136,37 +216,57 @@ class Router {
         if (label && this.routes.has(label)) {
             return this.routes.get(label);
         }
-        if (this.routes.has(defaultRoute)) {
-            return this.routes.get(defaultRoute);
+        if (this.routes.has(exports.defaultRoute)) {
+            return this.routes.get(exports.defaultRoute);
         }
         throw new errors_1.MissingRouteError(`Route not found for label '${String(label)}'.` +
             ' You must set up a route for this label or a default route.' +
             ' Use `requestHandler`, `router.addHandler` or `router.addDefaultHandler`.');
     }
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@link RequestValidationError} when validation fails.
+     */
+    async validateRequest(context) {
+        const label = context.request.label;
+        const schema = this.getSchema(label);
+        if (schema) {
+            context.request.userData = (await validateUserData(label, schema, context.request.userData));
+        }
+    }
     /**
      * Throws when the label already exists in our registry.
      */
     validate(label) {
         if (this.routes.has(label)) {
-            const message = label === defaultRoute
+            const message = label === exports.defaultRoute
                 ? `Default route is already defined!`
                 : `Route for label '${String(label)}' is already defined!`;
             throw new Error(message);
         }
     }
-    static create(routes) {
+    static create(routesOrSchemas) {
         const router = new Router();
         const obj = Object.create(Function.prototype);
         obj.addHandler = router.addHandler.bind(router);
         obj.addDefaultHandler = router.addDefaultHandler.bind(router);
+        obj.getSchema = router.getSchema.bind(router);
         obj.getHandler = router.getHandler.bind(router);
         obj.use = router.use.bind(router);
-        for (const [label, handler] of Object.entries(routes ?? {})) {
-            router.addHandler(label, handler);
+        // `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
+        for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {
+            const value = routesOrSchemas[label];
+            if (typeof value === 'function') {
+                router.addHandler(label, value);
+            }
+            else {
+                router.schemas.set(label, value);
+            }
         }
         const func = async function (context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
+            await router.validateRequest(context);
             for (const middleware of router.middlewares) {
                 await middleware(context);
             }